From b97fc7685aa768aaeb7718db80a3e77b6b79c89e Mon Sep 17 00:00:00 2001 From: master Date: Tue, 7 Oct 2025 10:14:21 +0300 Subject: [PATCH] Initial commit (history squashed) --- .../_deprecated-feedser-ci.yml.disabled | 29 + .../_deprecated-feedser-tests.yml.disabled | 87 + .gitea/workflows/build-test-deploy.yml | 341 ++ .gitea/workflows/docs.yml | 70 + .gitea/workflows/promote.yml | 206 + .gitignore | 21 + AGENTS.md | 125 + LICENSE | 235 ++ README.md | 28 + SPRINTS.md | 95 + TODOS.md | 12 + certificates/russian_trusted_sub_ca.cer | 10 + docs/01_WHAT_IS_IT.md | 77 + docs/02_WHY.md | 121 + docs/03_QUICKSTART.md | 156 + docs/03_VISION.md | 99 + docs/04_FEATURE_MATRIX.md | 34 + docs/05_ROADMAP.md | 6 + docs/05_SYSTEM_REQUIREMENTS_SPEC.md | 204 + docs/07_HIGH_LEVEL_ARCHITECTURE.md | 388 ++ docs/08_MODULE_SPECIFICATIONS.md | 208 + docs/09_API_CLI_REFERENCE.md | 461 +++ docs/10_FEEDSER_CLI_QUICKSTART.md | 289 ++ docs/10_OFFLINE_KIT.md | 139 + docs/10_PLUGIN_SDK_GUIDE.md | 194 + docs/11_DATA_SCHEMAS.md | 196 + docs/11_GOVERNANCE.md | 93 + docs/12_CODE_OF_CONDUCT.md | 88 + docs/12_PERFORMANCE_WORKBOOK.md | 167 + docs/13_RELEASE_ENGINEERING_PLAYBOOK.md | 209 + docs/13_SECURITY_POLICY.md | 101 + docs/14_GLOSSARY_OF_TERMS.md | 112 + docs/15_UI_GUIDE.md | 234 ++ docs/17_SECURITY_HARDENING_GUIDE.md | 201 + docs/18_CODING_STANDARDS.md | 169 + docs/19_TEST_SUITE_OVERVIEW.md | 109 + docs/21_INSTALL_GUIDE.md | 169 + docs/23_FAQ_MATRIX.md | 61 + docs/24_OFFLINE_KIT.md | 94 + docs/29_LEGAL_FAQ_QUOTA.md | 84 + docs/30_QUOTA_ENFORCEMENT_FLOW1.md | 93 + docs/33_333_QUOTA_OVERVIEW.md | 120 + docs/40_ARCHITECTURE_OVERVIEW.md | 133 + docs/60_POLICY_TEMPLATES.md | 101 + docs/AGENTS.md | 19 + docs/ARCHITECTURE_FEEDSER.md | 190 + docs/README.md | 67 + docs/TASKS.md | 12 + docs/_includes/CONSTANTS.md | 18 + docs/ci/20_CI_RECIPES.md | 258 ++ docs/cli/20_REFERENCE.md | 8 + docs/dev/30_PLUGIN_DEV_GUIDE.md | 146 + .../31_AUTHORITY_PLUGIN_DEVELOPER_GUIDE.md | 157 + docs/dev/32_AUTH_CLIENT_GUIDE.md | 91 + .../authority-rate-limit-tuning-outline.md | 33 + docs/dev/merge_semver_playbook.md | 99 + docs/dev/mongo_indices.md | 106 + docs/license-jwt-quota.md | 123 + docs/ops/authority-backup-restore.md | 97 + docs/ops/authority-grafana-dashboard.json | 174 + docs/ops/authority-monitoring.md | 81 + docs/ops/feedser-conflict-resolution.md | 130 + docs/ops/feedser-cve-kev-operations.md | 104 + docs/ops/migrations/SEMVER_STYLE.md | 50 + docs/rfcs/authority-plugin-ldap.md | 136 + etc/authority.plugins/ldap.yaml | 17 + etc/authority.plugins/standard.yaml | 22 + etc/authority.yaml.sample | 71 + etc/feedser.yaml.sample | 65 + ops/authority/AGENTS.md | 16 + ops/authority/Dockerfile | 38 + ops/authority/README.md | 39 + ops/authority/TASKS.md | 6 + ops/authority/docker-compose.authority.yaml | 58 + scripts/render_docs.py | 254 ++ scripts/update-model-goldens.ps1 | 9 + scripts/update-model-goldens.sh | 8 + src/DEDUP_CONFLICTS_RESOLUTION_ALGO.md | 143 + src/Directory.Build.props | 33 + src/Directory.Build.targets | 33 + src/FASTER_MODELING_AND_NORMALIZATION.md | 139 + src/StellaOps.Authority/AGENTS.md | 20 + .../NetworkMaskMatcherTests.cs | 75 + .../StellaOps.Auth.Abstractions.Tests.csproj | 10 + .../StellaOpsPrincipalBuilderTests.cs | 74 + .../StellaOpsProblemResultFactoryTests.cs | 53 + .../AuthorityTelemetry.cs | 56 + .../NetworkMask.cs | 181 + .../NetworkMaskMatcher.cs | 139 + .../README.NuGet.md | 9 + .../StellaOps.Auth.Abstractions.csproj | 37 + .../StellaOpsAuthenticationDefaults.cs | 22 + .../StellaOpsClaimTypes.cs | 57 + .../StellaOpsPrincipalBuilder.cs | 287 ++ .../StellaOpsProblemResultFactory.cs | 114 + .../StellaOpsScopes.cs | 79 + .../ServiceCollectionExtensionsTests.cs | 95 + .../StellaOps.Auth.Client.Tests.csproj | 15 + .../StellaOpsAuthClientOptionsTests.cs | 84 + .../StellaOpsDiscoveryCacheTests.cs | 134 + .../StellaOpsTokenClientTests.cs | 111 + .../TokenCacheTests.cs | 59 + .../StellaOps.Auth.Client/FileTokenCache.cs | 122 + .../IStellaOpsTokenCache.cs | 25 + .../IStellaOpsTokenClient.cs | 41 + .../InMemoryTokenCache.cs | 58 + .../StellaOps.Auth.Client/README.NuGet.md | 9 + .../ServiceCollectionExtensions.cs | 115 + .../StellaOps.Auth.Client.csproj | 46 + .../StellaOpsAuthClientOptions.cs | 205 + .../StellaOpsDiscoveryCache.cs | 142 + .../StellaOpsJwksCache.cs | 116 + .../StellaOpsTokenCacheEntry.cs | 49 + .../StellaOpsTokenClient.cs | 205 + .../StellaOpsTokenResult.cs | 23 + .../ServiceCollectionExtensionsTests.cs | 44 + ...llaOps.Auth.ServerIntegration.Tests.csproj | 11 + .../StellaOpsResourceServerOptionsTests.cs | 50 + ...StellaOpsScopeAuthorizationHandlerTests.cs | 123 + .../README.NuGet.md | 9 + .../ServiceCollectionExtensions.cs | 88 + .../StellaOps.Auth.ServerIntegration.csproj | 46 + ...OpsAuthorizationPolicyBuilderExtensions.cs | 56 + .../StellaOpsBypassEvaluator.cs | 62 + .../StellaOpsResourceServerOptions.cs | 152 + .../StellaOpsScopeAuthorizationHandler.cs | 111 + .../StellaOpsScopeRequirement.cs | 47 + .../StandardClientProvisioningStoreTests.cs | 66 + .../StandardPluginOptionsTests.cs | 99 + .../StandardPluginRegistrarTests.cs | 227 ++ .../StandardUserCredentialStoreTests.cs | 102 + ...Ops.Authority.Plugin.Standard.Tests.csproj | 12 + .../AGENTS.md | 20 + .../Bootstrap/StandardPluginBootstrapper.cs | 42 + .../Properties/AssemblyInfo.cs | 3 + .../Security/IPasswordHasher.cs | 113 + .../StandardClaimsEnricher.cs | 43 + .../StandardIdentityProviderPlugin.cs | 65 + .../StandardPluginOptions.cs | 130 + .../StandardPluginRegistrar.cs | 88 + ...StellaOps.Authority.Plugin.Standard.csproj | 22 + .../StandardClientProvisioningStore.cs | 109 + .../Storage/StandardUserCredentialStore.cs | 329 ++ .../Storage/StandardUserDocument.cs | 64 + .../TASKS.md | 16 + .../AuthorityClientRegistrationTests.cs | 31 + ...horityCredentialVerificationResultTests.cs | 38 + ...horityIdentityProviderCapabilitiesTests.cs | 42 + .../AuthorityPluginHealthResultTests.cs | 32 + .../AuthorityPluginOperationResultTests.cs | 60 + .../AuthorityUserDescriptorTests.cs | 28 + .../AuthorityUserRegistrationTests.cs | 25 + ...uthority.Plugins.Abstractions.Tests.csproj | 11 + .../AuthorityClientMetadataKeys.cs | 12 + .../AuthorityPluginContracts.cs | 139 + .../AuthorityPluginRegistrationContext.cs | 60 + .../AuthoritySecretHasher.cs | 25 + .../IdentityProviderContracts.cs | 785 ++++ ...aOps.Authority.Plugins.Abstractions.csproj | 14 + .../AuthorityMongoDefaults.cs | 24 + .../Class1.cs | 6 + .../Documents/AuthorityClientDocument.cs | 61 + .../AuthorityLoginAttemptDocument.cs | 45 + .../Documents/AuthorityScopeDocument.cs | 38 + .../Documents/AuthorityTokenDocument.cs | 54 + .../Documents/AuthorityUserDocument.cs | 51 + .../Extensions/ServiceCollectionExtensions.cs | 103 + .../AuthorityClientCollectionInitializer.cs | 24 + ...horityLoginAttemptCollectionInitializer.cs | 26 + .../AuthorityMongoInitializer.cs | 55 + .../AuthorityScopeCollectionInitializer.cs | 21 + .../AuthorityTokenCollectionInitializer.cs | 40 + .../AuthorityUserCollectionInitializer.cs | 27 + .../IAuthorityCollectionInitializer.cs | 14 + .../AuthorityMongoMigrationRunner.cs | 40 + .../EnsureAuthorityCollectionsMigration.cs | 44 + .../Migrations/IAuthorityMongoMigration.cs | 16 + .../Options/AuthorityMongoOptions.cs | 64 + .../StellaOps.Authority.Storage.Mongo.csproj | 18 + .../Stores/AuthorityClientStore.cs | 64 + .../Stores/AuthorityLoginAttemptStore.cs | 51 + .../Stores/AuthorityScopeStore.cs | 69 + .../Stores/AuthorityTokenStore.cs | 93 + .../Stores/AuthorityUserStore.cs | 81 + .../Stores/IAuthorityClientStore.cs | 12 + .../Stores/IAuthorityLoginAttemptStore.cs | 10 + .../Stores/IAuthorityScopeStore.cs | 14 + .../Stores/IAuthorityTokenStore.cs | 16 + .../Stores/IAuthorityUserStore.cs | 14 + .../AuthorityIdentityProviderRegistryTests.cs | 125 + .../AuthorityIdentityProviderSelectorTests.cs | 118 + .../ClientCredentialsAndTokenHandlersTests.cs | 449 +++ .../TokenPersistenceIntegrationTests.cs | 208 + .../Plugins/AuthorityPluginLoaderTests.cs | 117 + .../AuthorityRateLimiterIntegrationTests.cs | 141 + ...thorityRateLimiterMetadataAccessorTests.cs | 30 + ...orityRateLimiterMetadataMiddlewareTests.cs | 96 + .../RateLimiting/AuthorityRateLimiterTests.cs | 80 + .../StellaOps.Authority.Tests.csproj | 12 + .../StellaOps.Authority.sln | 370 ++ .../AuthorityIdentityProviderRegistry.cs | 103 + .../AuthorityPluginRegistry.cs | 23 + .../AuthorityRateLimiter.cs | 117 + .../AuthorityTelemetryConfiguration.cs | 60 + .../Bootstrap/BootstrapApiKeyFilter.cs | 30 + .../Bootstrap/BootstrapRequests.cs | 48 + .../AuthorityIdentityProviderSelector.cs | 63 + .../AuthorityOpenIddictConstants.cs | 11 + .../Handlers/ClientCredentialsHandlers.cs | 394 ++ .../Handlers/PasswordGrantHandlers.cs | 187 + .../Handlers/TokenValidationHandlers.cs | 147 + .../Plugins/AuthorityPluginLoader.cs | 225 ++ .../AuthorityPluginRegistrationSummary.cs | 20 + .../StellaOps.Authority/Program.cs | 491 +++ .../Properties/AssemblyInfo.cs | 3 + .../Properties/launchSettings.json | 23 + .../AuthorityRateLimiterFeature.cs | 26 + .../AuthorityRateLimiterMetadata.cs | 64 + .../AuthorityRateLimiterMetadataAccessor.cs | 83 + .../AuthorityRateLimiterMetadataMiddleware.cs | 278 ++ ...uthorityRateLimiterPartitionKeyResolver.cs | 64 + ...ateLimitingApplicationBuilderExtensions.cs | 13 + .../StellaOps.Authority.csproj | 28 + .../appsettings.Development.json | 8 + .../StellaOps.Authority/appsettings.json | 9 + src/StellaOps.Authority/TASKS.md | 16 + .../Commands/CommandHandlersTests.cs | 533 +++ .../Configuration/CliBootstrapperTests.cs | 110 + .../Services/BackendOperationsClientTests.cs | 417 ++ .../StellaOps.Cli.Tests.csproj | 28 + .../Testing/TestHelpers.cs | 55 + src/StellaOps.Cli.Tests/UnitTest1.cs | 10 + src/StellaOps.Cli.Tests/xunit.runner.json | 3 + src/StellaOps.Cli/AGENTS.md | 27 + src/StellaOps.Cli/Commands/CommandFactory.cs | 324 ++ src/StellaOps.Cli/Commands/CommandHandlers.cs | 840 ++++ .../Configuration/AuthorityTokenUtilities.cs | 34 + .../Configuration/CliBootstrapper.cs | 278 ++ .../Configuration/StellaOpsCliOptions.cs | 56 + src/StellaOps.Cli/Program.cs | 119 + .../Prompts/TrivyDbExportPrompt.cs | 52 + src/StellaOps.Cli/Properties/AssemblyInfo.cs | 3 + .../Services/BackendOperationsClient.cs | 535 +++ .../Services/IBackendOperationsClient.cs | 16 + .../Services/IScannerExecutor.cs | 17 + .../Services/IScannerInstaller.cs | 9 + .../Services/Models/JobTriggerResult.cs | 9 + .../Services/Models/ScannerArtifactResult.cs | 3 + .../Models/Transport/JobRunResponse.cs | 27 + .../Models/Transport/JobTriggerRequest.cs | 10 + .../Models/Transport/ProblemDocument.cs | 18 + .../Services/ScannerExecutionResult.cs | 3 + src/StellaOps.Cli/Services/ScannerExecutor.cs | 329 ++ .../Services/ScannerInstaller.cs | 79 + src/StellaOps.Cli/StellaOps.Cli.csproj | 44 + src/StellaOps.Cli/TASKS.md | 14 + .../Telemetry/CliActivitySource.cs | 8 + src/StellaOps.Cli/Telemetry/CliMetrics.cs | 62 + src/StellaOps.Cli/Telemetry/VerbosityState.cs | 8 + src/StellaOps.Cli/appsettings.json | 11 + ...AuthorityPluginConfigurationLoaderTests.cs | 126 + .../AuthorityTelemetryTests.cs | 24 + .../StellaOps.Configuration.Tests.csproj | 11 + .../StellaOpsAuthorityOptionsTests.cs | 148 + .../AuthorityPluginConfigurationLoader.cs | 100 + .../StellaOps.Configuration.csproj | 23 + .../StellaOpsAuthorityConfiguration.cs | 57 + .../StellaOpsAuthorityOptions.cs | 529 +++ .../StellaOpsBootstrapOptions.cs | 64 + .../StellaOpsConfigurationBootstrapper.cs | 106 + .../StellaOpsConfigurationContext.cs | 18 + .../StellaOpsConfigurationOptions.cs | 49 + .../StellaOpsOptionsBinder.cs | 26 + .../PasswordHashOptionsTests.cs | 24 + .../StellaOps.Cryptography.Tests.csproj | 11 + src/StellaOps.Cryptography/AGENTS.md | 21 + src/StellaOps.Cryptography/CryptoProvider.cs | 44 + src/StellaOps.Cryptography/PasswordHashing.cs | 81 + .../StellaOps.Cryptography.csproj | 9 + src/StellaOps.Cryptography/TASKS.md | 25 + .../IDependencyInjectionRoutine.cs | 11 + .../StellaOps.DependencyInjection.csproj | 14 + .../CanonicalMergerTests.cs | 239 ++ .../JobCoordinatorTests.cs | 483 +++ .../JobPluginRegistrationExtensionsTests.cs | 61 + .../JobSchedulerBuilderTests.cs | 70 + .../PluginRoutineFixtures.cs | 42 + .../StellaOps.Feedser.Core.Tests.csproj | 10 + src/StellaOps.Feedser.Core/AGENTS.md | 32 + .../CanonicalMergeResult.cs | 19 + src/StellaOps.Feedser.Core/CanonicalMerger.cs | 669 ++++ src/StellaOps.Feedser.Core/Jobs/IJob.cs | 6 + .../Jobs/IJobCoordinator.cs | 18 + src/StellaOps.Feedser.Core/Jobs/IJobStore.cs | 20 + .../Jobs/ILeaseStore.cs | 10 + .../Jobs/JobCoordinator.cs | 635 ++++ .../Jobs/JobDefinition.cs | 12 + .../Jobs/JobDiagnostics.cs | 171 + .../Jobs/JobExecutionContext.cs | 42 + src/StellaOps.Feedser.Core/Jobs/JobLease.cs | 9 + .../Jobs/JobPluginRegistrationExtensions.cs | 128 + .../Jobs/JobRunCompletion.cs | 6 + .../Jobs/JobRunCreateRequest.cs | 10 + .../Jobs/JobRunSnapshot.cs | 21 + .../Jobs/JobRunStatus.cs | 10 + .../Jobs/JobSchedulerBuilder.cs | 47 + .../Jobs/JobSchedulerHostedService.cs | 165 + .../Jobs/JobSchedulerOptions.cs | 12 + .../Jobs/JobTriggerResult.cs | 40 + .../Jobs/ServiceCollectionExtensions.cs | 27 + .../StellaOps.Feedser.Core.csproj | 19 + src/StellaOps.Feedser.Core/TASKS.md | 16 + .../JsonExportSnapshotBuilderTests.cs | 213 ++ ...ExporterDependencyInjectionRoutineTests.cs | 83 + .../JsonExporterParitySmokeTests.cs | 182 + .../JsonFeedExporterTests.cs | 265 ++ ...ellaOps.Feedser.Exporter.Json.Tests.csproj | 13 + .../VulnListJsonExportPathResolverTests.cs | 148 + src/StellaOps.Feedser.Exporter.Json/AGENTS.md | 28 + .../ExportDigestCalculator.cs | 52 + .../ExporterVersion.cs | 28 + .../IJsonExportPathResolver.cs | 12 + .../JsonExportFile.cs | 37 + .../JsonExportJob.cs | 30 + .../JsonExportManifestWriter.cs | 66 + .../JsonExportOptions.cs | 34 + .../JsonExportResult.cs | 46 + .../JsonExportSnapshotBuilder.cs | 239 ++ .../JsonExporterDependencyInjectionRoutine.cs | 59 + .../JsonExporterPlugin.cs | 23 + .../JsonFeedExporter.cs | 170 + .../StellaOps.Feedser.Exporter.Json.csproj | 22 + src/StellaOps.Feedser.Exporter.Json/TASKS.md | 11 + .../VulnListJsonExportPathResolver.cs | 455 +++ ...aOps.Feedser.Exporter.TrivyDb.Tests.csproj | 13 + .../TrivyDbExportPlannerTests.cs | 86 + .../TrivyDbFeedExporterTests.cs | 1027 +++++ .../TrivyDbOciWriterTests.cs | 149 + .../TrivyDbPackageBuilderTests.cs | 93 + .../AGENTS.md | 29 + .../ITrivyDbBuilder.cs | 15 + .../ITrivyDbOrasPusher.cs | 9 + .../OciDescriptor.cs | 10 + .../OciIndex.cs | 8 + .../OciManifest.cs | 10 + .../StellaOps.Feedser.Exporter.TrivyDb.csproj | 22 + .../TASKS.md | 13 + .../TrivyConfigDocument.cs | 11 + .../TrivyDbBlob.cs | 78 + .../TrivyDbBoltBuilder.cs | 376 ++ .../TrivyDbBuilderResult.cs | 10 + .../TrivyDbExportJob.cs | 94 + .../TrivyDbExportMode.cs | 8 + .../TrivyDbExportOptions.cs | 88 + .../TrivyDbExportOverrides.cs | 50 + .../TrivyDbExportPlan.cs | 14 + .../TrivyDbExportPlanner.cs | 115 + ...ivyDbExporterDependencyInjectionRoutine.cs | 64 + .../TrivyDbExporterPlugin.cs | 23 + .../TrivyDbFeedExporter.cs | 490 +++ .../TrivyDbMediaTypes.cs | 9 + .../TrivyDbOciWriteResult.cs | 8 + .../TrivyDbOciWriter.cs | 375 ++ .../TrivyDbOrasPusher.cs | 209 + .../TrivyDbPackage.cs | 9 + .../TrivyDbPackageBuilder.cs | 116 + .../TrivyDbPackageRequest.cs | 11 + .../AdvisoryIdentityResolverTests.cs | 92 + .../AdvisoryMergeServiceTests.cs | 201 + .../AdvisoryPrecedenceMergerTests.cs | 385 ++ .../AffectedPackagePrecedenceResolverTests.cs | 96 + .../AliasGraphResolverTests.cs | 135 + .../CanonicalHashCalculatorTests.cs | 86 + .../DebianEvrComparerTests.cs | 84 + .../MergeEventWriterTests.cs | 85 + .../MergePrecedenceIntegrationTests.cs | 231 ++ .../MetricCollector.cs | 56 + .../NevraComparerTests.cs | 108 + .../SemanticVersionRangeResolverTests.cs | 67 + .../StellaOps.Feedser.Merge.Tests.csproj | 13 + .../TestLogger.cs | 52 + src/StellaOps.Feedser.Merge/AGENTS.md | 33 + src/StellaOps.Feedser.Merge/Class1.cs | 1 + .../Comparers/DebianEvr.cs | 232 ++ .../Comparers/Nevra.cs | 264 ++ .../Comparers/SemanticVersionRangeResolver.cs | 73 + .../Identity/AdvisoryIdentityCluster.cs | 56 + .../Identity/AdvisoryIdentityResolver.cs | 303 ++ .../Identity/AliasIdentity.cs | 24 + .../Jobs/MergeJobKinds.cs | 6 + .../Jobs/MergeReconcileJob.cs | 43 + .../MergeServiceCollectionExtensions.cs | 43 + .../Options/AdvisoryPrecedenceDefaults.cs | 96 + .../Options/AdvisoryPrecedenceOptions.cs | 15 + .../Options/AdvisoryPrecedenceTable.cs | 35 + .../RANGE_PRIMITIVES_COORDINATION.md | 44 + .../Services/AdvisoryMergeService.cs | 291 ++ .../Services/AdvisoryPrecedenceMerger.cs | 520 +++ .../AffectedPackagePrecedenceResolver.cs | 163 + .../Services/AliasGraphResolver.cs | 139 + .../Services/CanonicalHashCalculator.cs | 25 + .../Services/MergeEventWriter.cs | 72 + .../StellaOps.Feedser.Merge.csproj | 18 + src/StellaOps.Feedser.Merge/TASKS.md | 18 + .../AdvisoryProvenanceTests.cs | 49 + .../AdvisoryTests.cs | 62 + .../AffectedPackageStatusTests.cs | 75 + .../AliasSchemeRegistryTests.cs | 52 + .../CanonicalExampleFactory.cs | 195 + .../CanonicalExamplesTests.cs | 60 + .../CanonicalJsonSerializerTests.cs | 152 + .../Fixtures/ghsa-semver.json | 124 + .../Fixtures/kev-flag.json | 42 + .../Fixtures/nvd-basic.json | 119 + .../Fixtures/psirt-overlay.json | 122 + .../NormalizedVersionRuleTests.cs | 68 + .../OsvGhsaParityDiagnosticsTests.cs | 88 + .../OsvGhsaParityInspectorTests.cs | 148 + .../ProvenanceDiagnosticsTests.cs | 180 + .../RangePrimitivesTests.cs | 41 + .../SemVerPrimitiveTests.cs | 132 + .../SerializationDeterminismTests.cs | 68 + .../SeverityNormalizationTests.cs | 36 + .../StellaOps.Feedser.Models.Tests.csproj | 15 + src/StellaOps.Feedser.Models/AGENTS.md | 30 + src/StellaOps.Feedser.Models/Advisory.cs | 189 + .../AdvisoryCredit.cs | 101 + .../AdvisoryProvenance.cs | 70 + .../AdvisoryReference.cs | 36 + .../AffectedPackage.cs | 97 + .../AffectedPackageStatus.cs | 46 + .../AffectedPackageStatusCatalog.cs | 156 + .../AffectedVersionRange.cs | 149 + .../AffectedVersionRangeExtensions.cs | 75 + .../AliasSchemeRegistry.cs | 166 + src/StellaOps.Feedser.Models/AliasSchemes.cs | 31 + .../BACKWARD_COMPATIBILITY.md | 41 + .../CANONICAL_RECORDS.md | 144 + .../CanonicalJsonSerializer.cs | 161 + src/StellaOps.Feedser.Models/CvssMetric.cs | 31 + .../NormalizedVersionRule.cs | 185 + .../OsvGhsaParityDiagnostics.cs | 72 + .../OsvGhsaParityInspector.cs | 183 + .../PROVENANCE_GUIDELINES.md | 15 + .../ProvenanceFieldMasks.cs | 16 + .../ProvenanceInspector.cs | 297 ++ .../RangePrimitives.cs | 161 + .../SemVerPrimitiveExtensions.cs | 88 + .../SeverityNormalization.cs | 152 + .../SnapshotSerializer.cs | 27 + .../StellaOps.Feedser.Models.csproj | 12 + src/StellaOps.Feedser.Models/TASKS.md | 18 + src/StellaOps.Feedser.Models/Validation.cs | 57 + .../CpeNormalizerTests.cs | 70 + .../CvssMetricNormalizerTests.cs | 52 + .../DebianEvrParserTests.cs | 31 + .../DescriptionNormalizerTests.cs | 44 + .../NevraParserTests.cs | 64 + .../PackageUrlNormalizerTests.cs | 44 + .../SemVerRangeRuleBuilderTests.cs | 113 + ...ellaOps.Feedser.Normalization.Tests.csproj | 11 + .../AssemblyInfo.cs | 8 + .../Cvss/CvssMetricNormalizer.cs | 529 +++ .../Distro/DebianEvr.cs | 127 + .../Distro/Nevra.cs | 192 + .../Identifiers/Cpe23.cs | 352 ++ .../Identifiers/IdentifierNormalizer.cs | 32 + .../Identifiers/PackageUrl.cs | 299 ++ .../SemVer/SemVerRangeRuleBuilder.cs | 615 +++ .../StellaOps.Feedser.Normalization.csproj | 22 + src/StellaOps.Feedser.Normalization/TASKS.md | 8 + .../Text/DescriptionNormalizer.cs | 118 + src/StellaOps.Feedser.Source.Acsc/AGENTS.md | 40 + src/StellaOps.Feedser.Source.Acsc/Class1.cs | 29 + .../StellaOps.Feedser.Source.Acsc.csproj | 16 + src/StellaOps.Feedser.Source.Acsc/TASKS.md | 11 + src/StellaOps.Feedser.Source.Cccs/AGENTS.md | 40 + src/StellaOps.Feedser.Source.Cccs/Class1.cs | 29 + .../StellaOps.Feedser.Source.Cccs.csproj | 16 + src/StellaOps.Feedser.Source.Cccs/TASKS.md | 10 + .../AGENTS.md | 40 + .../Class1.cs | 29 + .../StellaOps.Feedser.Source.CertBund.csproj | 16 + .../TASKS.md | 10 + .../CertCc/CertCcConnectorFetchTests.cs | 263 ++ .../CertCc/CertCcConnectorSnapshotTests.cs | 405 ++ .../CertCc/CertCcConnectorTests.cs | 404 ++ .../Fixtures/certcc-documents.snapshot.json | 91 + .../Fixtures/certcc-state.snapshot.json | 13 + .../Fixtures/summary-2025-09.json | 4 + .../Fixtures/summary-2025-10.json | 6 + .../Fixtures/summary-2025-11.json | 4 + .../Fixtures/summary-2025.json | 6 + .../Fixtures/vendor-statuses-294418.json | 11 + .../Fixtures/vendors-294418.json | 12 + .../Fixtures/vu-257161.json | 87 + .../Fixtures/vu-294418-vendors.json | 12 + .../Fixtures/vu-294418-vuls.json | 11 + .../Fixtures/vu-294418.json | 63 + .../Fixtures/vulnerabilities-294418.json | 11 + .../Internal/CertCcMapperTests.cs | 118 + .../Internal/CertCcSummaryPlannerTests.cs | 95 + .../CertCcVendorStatementParserTests.cs | 31 + ...ellaOps.Feedser.Source.CertCc.Tests.csproj | 16 + src/StellaOps.Feedser.Source.CertCc/AGENTS.md | 38 + .../CertCcConnector.cs | 124 + .../CertCcConnectorPlugin.cs | 21 + .../CertCcDependencyInjectionRoutine.cs | 50 + .../CertCcServiceCollectionExtensions.cs | 36 + .../Configuration/CertCcOptions.cs | 54 + .../FEEDCONN-CERTCC-02-009_PLAN.md | 59 + .../Internal/CertCcCursor.cs | 58 + .../Internal/CertCcDiagnostics.cs | 90 + .../Internal/CertCcMapper.cs | 607 +++ .../Internal/CertCcNoteDto.cs | 97 + .../Internal/CertCcNoteParser.cs | 539 +++ .../Internal/CertCcSummaryPlan.cs | 22 + .../Internal/CertCcSummaryPlanner.cs | 96 + .../Internal/CertCcVendorStatementParser.cs | 235 ++ src/StellaOps.Feedser.Source.CertCc/Jobs.cs | 22 + .../Properties/AssemblyInfo.cs | 3 + src/StellaOps.Feedser.Source.CertCc/README.md | 38 + .../StellaOps.Feedser.Source.CertCc.csproj | 16 + src/StellaOps.Feedser.Source.CertCc/TASKS.md | 12 + .../CertFr/CertFrConnectorTests.cs | 313 ++ .../Fixtures/certfr-advisories.snapshot.json | 205 + .../Fixtures/certfr-detail-AV-2024-001.html | 8 + .../Fixtures/certfr-detail-AV-2024-002.html | 11 + .../CertFr/Fixtures/certfr-feed.xml | 22 + ...ellaOps.Feedser.Source.CertFr.Tests.csproj | 16 + src/StellaOps.Feedser.Source.CertFr/AGENTS.md | 27 + .../CertFrConnector.cs | 337 ++ .../CertFrConnectorPlugin.cs | 21 + .../CertFrDependencyInjectionRoutine.cs | 54 + .../CertFrServiceCollectionExtensions.cs | 36 + .../Configuration/CertFrOptions.cs | 46 + .../Internal/CertFrCursor.cs | 88 + .../Internal/CertFrDocumentMetadata.cs | 77 + .../Internal/CertFrDto.cs | 14 + .../Internal/CertFrFeedClient.cs | 109 + .../Internal/CertFrFeedItem.cs | 10 + .../Internal/CertFrMapper.cs | 116 + .../Internal/CertFrParser.cs | 80 + src/StellaOps.Feedser.Source.CertFr/Jobs.cs | 46 + .../StellaOps.Feedser.Source.CertFr.csproj | 13 + src/StellaOps.Feedser.Source.CertFr/TASKS.md | 11 + .../CertIn/CertInConnectorTests.cs | 350 ++ .../CertIn/Fixtures/alerts-page1.json | 9 + .../Fixtures/detail-CIAD-2024-0005.html | 17 + .../CertIn/Fixtures/expected-advisory.json | 128 + ...ellaOps.Feedser.Source.CertIn.Tests.csproj | 16 + src/StellaOps.Feedser.Source.CertIn/AGENTS.md | 28 + .../CertInConnector.cs | 462 +++ .../CertInConnectorPlugin.cs | 19 + .../CertInDependencyInjectionRoutine.cs | 54 + .../CertInServiceCollectionExtensions.cs | 37 + .../Configuration/CertInOptions.cs | 68 + .../Internal/CertInAdvisoryDto.cs | 16 + .../Internal/CertInClient.cs | 129 + .../Internal/CertInCursor.cs | 88 + .../Internal/CertInDetailParser.cs | 187 + .../Internal/CertInListingItem.cs | 10 + src/StellaOps.Feedser.Source.CertIn/Jobs.cs | 46 + .../StellaOps.Feedser.Source.CertIn.csproj | 16 + src/StellaOps.Feedser.Source.CertIn/TASKS.md | 10 + .../Common/CannedHttpMessageHandlerTests.cs | 37 + .../Common/HtmlContentSanitizerTests.cs | 31 + .../Common/PackageCoordinateHelperTests.cs | 41 + .../Common/PdfTextExtractorTests.cs | 21 + .../Common/SourceFetchServiceTests.cs | 36 + .../Common/TimeWindowCursorPlannerTests.cs | 87 + .../Common/UrlNormalizerTests.cs | 24 + .../Json/JsonSchemaValidatorTests.cs | 51 + ...ellaOps.Feedser.Source.Common.Tests.csproj | 10 + .../Xml/XmlSchemaValidatorTests.cs | 58 + src/StellaOps.Feedser.Source.Common/AGENTS.md | 31 + .../Cursors/PaginationPlanner.cs | 29 + .../Cursors/TimeWindowCursorOptions.cs | 43 + .../Cursors/TimeWindowCursorPlanner.cs | 50 + .../Cursors/TimeWindowCursorState.cs | 84 + .../DocumentStatuses.cs | 27 + .../Fetch/CryptoJitterSource.cs | 43 + .../Fetch/IJitterSource.cs | 9 + .../Fetch/RawDocumentStorage.cs | 90 + .../Fetch/SourceFetchContentResult.cs | 58 + .../Fetch/SourceFetchRequest.cs | 24 + .../Fetch/SourceFetchResult.cs | 34 + .../Fetch/SourceFetchService.cs | 313 ++ .../Fetch/SourceRetryPolicy.cs | 79 + .../Html/HtmlContentSanitizer.cs | 180 + .../Http/AllowlistedHttpMessageHandler.cs | 36 + .../Http/ServiceCollectionExtensions.cs | 76 + .../Http/SourceHttpClientOptions.cs | 80 + .../Json/IJsonSchemaValidator.cs | 9 + .../Json/JsonSchemaValidationError.cs | 7 + .../Json/JsonSchemaValidationException.cs | 15 + .../Json/JsonSchemaValidator.cs | 92 + .../Packages/PackageCoordinateHelper.cs | 197 + .../Pdf/PdfTextExtractor.cs | 184 + .../Properties/AssemblyInfo.cs | 3 + .../StellaOps.Feedser.Source.Common.csproj | 21 + src/StellaOps.Feedser.Source.Common/TASKS.md | 16 + .../Telemetry/SourceDiagnostics.cs | 107 + .../Testing/CannedHttpMessageHandler.cs | 210 + .../Url/UrlNormalizer.cs | 62 + .../Xml/IXmlSchemaValidator.cs | 9 + .../Xml/XmlSchemaValidationError.cs | 3 + .../Xml/XmlSchemaValidationException.cs | 18 + .../Xml/XmlSchemaValidator.cs | 71 + .../Cve/CveConnectorTests.cs | 130 + .../Fixtures/cve-CVE-2024-0001.json | 72 + .../Fixtures/cve-list.json | 18 + .../Fixtures/expected-CVE-2024-0001.json | 163 + .../StellaOps.Feedser.Source.Cve.Tests.csproj | 17 + src/StellaOps.Feedser.Source.Cve/AGENTS.md | 38 + .../Configuration/CveOptions.cs | 100 + .../CveConnector.cs | 398 ++ .../CveConnectorPlugin.cs | 19 + .../CveDependencyInjectionRoutine.cs | 54 + .../CveServiceCollectionExtensions.cs | 38 + .../Internal/CveCursor.cs | 135 + .../Internal/CveDiagnostics.cs | 74 + .../Internal/CveListParser.cs | 264 ++ .../Internal/CveMapper.cs | 278 ++ .../Internal/CveRecordDto.cs | 105 + .../Internal/CveRecordParser.cs | 346 ++ src/StellaOps.Feedser.Source.Cve/Jobs.cs | 43 + .../StellaOps.Feedser.Source.Cve.csproj | 16 + src/StellaOps.Feedser.Source.Cve/TASKS.md | 11 + .../DebianConnectorTests.cs | 281 ++ .../DebianMapperTests.cs | 88 + .../Fixtures/debian-detail-dsa-2024-123.html | 23 + .../Fixtures/debian-detail-dsa-2024-124.html | 21 + .../Distro/Debian/Fixtures/debian-list.txt | 7 + ....Feedser.Source.Distro.Debian.Tests.csproj | 13 + .../AssemblyInfo.cs | 3 + .../Configuration/DebianOptions.cs | 87 + .../DebianConnector.cs | 637 ++++ .../DebianConnectorPlugin.cs | 22 + .../DebianDependencyInjectionRoutine.cs | 53 + .../DebianServiceCollectionExtensions.cs | 37 + .../Internal/DebianAdvisoryDto.cs | 27 + .../Internal/DebianCursor.cs | 177 + .../Internal/DebianDetailMetadata.cs | 12 + .../Internal/DebianFetchCacheEntry.cs | 76 + .../Internal/DebianHtmlParser.cs | 326 ++ .../Internal/DebianListEntry.cs | 11 + .../Internal/DebianListParser.cs | 107 + .../Internal/DebianMapper.cs | 266 ++ .../Jobs.cs | 46 + ...llaOps.Feedser.Source.Distro.Debian.csproj | 17 + .../RedHat/Fixtures/csaf-rhsa-2025-0001.json | 95 + .../RedHat/Fixtures/csaf-rhsa-2025-0002.json | 82 + .../RedHat/Fixtures/csaf-rhsa-2025-0003.json | 93 + .../Fixtures/rhsa-2025-0001.snapshot.json | 149 + .../Fixtures/rhsa-2025-0002.snapshot.json | 118 + .../Fixtures/rhsa-2025-0003.snapshot.json | 121 + .../RedHat/Fixtures/summary-page1-repeat.json | 14 + .../RedHat/Fixtures/summary-page1.json | 14 + .../RedHat/Fixtures/summary-page2.json | 8 + .../RedHat/Fixtures/summary-page3.json | 8 + .../RedHat/RedHatConnectorHarnessTests.cs | 123 + .../RedHat/RedHatConnectorTests.cs | 653 ++++ ....Feedser.Source.Distro.RedHat.Tests.csproj | 18 + .../AGENTS.md | 27 + .../CONFLICT_RESOLVER_NOTES.md | 25 + .../Configuration/RedHatOptions.cs | 97 + .../Internal/Models/RedHatCsafModels.cs | 177 + .../Internal/RedHatCursor.cs | 254 ++ .../Internal/RedHatMapper.cs | 758 ++++ .../Internal/RedHatSummaryItem.cs | 66 + .../Jobs.cs | 46 + .../Properties/AssemblyInfo.cs | 3 + .../RedHatConnector.cs | 434 +++ .../RedHatConnectorPlugin.cs | 19 + .../RedHatDependencyInjectionRoutine.cs | 54 + .../RedHatServiceCollectionExtensions.cs | 34 + ...llaOps.Feedser.Source.Distro.RedHat.csproj | 15 + .../TASKS.md | 16 + .../Distro/Suse/Fixtures/suse-changes.csv | 2 + .../Suse/Fixtures/suse-su-2025_0001-1.json | 63 + .../Suse/Fixtures/suse-su-2025_0002-1.json | 66 + ...ps.Feedser.Source.Distro.Suse.Tests.csproj | 18 + .../SuseConnectorTests.cs | 168 + .../SuseCsafParserTests.cs | 52 + .../SuseMapperTests.cs | 52 + .../AssemblyInfo.cs | 3 + .../Configuration/SuseOptions.cs | 86 + .../Internal/SuseAdvisoryDto.cs | 28 + .../Internal/SuseChangeRecord.cs | 5 + .../Internal/SuseChangesParser.cs | 81 + .../Internal/SuseCsafParser.cs | 422 +++ .../Internal/SuseCursor.cs | 177 + .../Internal/SuseFetchCacheEntry.cs | 76 + .../Internal/SuseMapper.cs | 313 ++ .../Jobs.cs | 46 + ...tellaOps.Feedser.Source.Distro.Suse.csproj | 17 + .../SuseConnector.cs | 573 +++ .../SuseConnectorPlugin.cs | 20 + .../SuseDependencyInjectionRoutine.cs | 53 + .../SuseServiceCollectionExtensions.cs | 35 + .../Fixtures/ubuntu-notices-page0.json | 40 + .../Fixtures/ubuntu-notices-page1.json | 42 + ....Feedser.Source.Distro.Ubuntu.Tests.csproj | 18 + .../UbuntuConnectorTests.cs | 171 + .../Configuration/UbuntuOptions.cs | 69 + .../Internal/UbuntuCursor.cs | 177 + .../Internal/UbuntuFetchCacheEntry.cs | 76 + .../Internal/UbuntuMapper.cs | 217 ++ .../Internal/UbuntuNoticeDto.cs | 25 + .../Internal/UbuntuNoticeParser.cs | 215 ++ .../Jobs.cs | 46 + ...llaOps.Feedser.Source.Distro.Ubuntu.csproj | 17 + .../TASKS.md | 9 + .../UbuntuConnector.cs | 537 +++ .../UbuntuConnectorPlugin.cs | 20 + .../UbuntuDependencyInjectionRoutine.cs | 53 + .../UbuntuServiceCollectionExtensions.cs | 37 + .../expected-GHSA-xxxx-yyyy-zzzz.json | 113 + .../Fixtures/ghsa-GHSA-xxxx-yyyy-zzzz.json | 50 + .../Fixtures/ghsa-list.json | 12 + .../Ghsa/GhsaConnectorTests.cs | 135 + ...StellaOps.Feedser.Source.Ghsa.Tests.csproj | 17 + src/StellaOps.Feedser.Source.Ghsa/AGENTS.md | 39 + .../Configuration/GhsaOptions.cs | 61 + .../GhsaConnector.cs | 394 ++ .../GhsaConnectorPlugin.cs | 19 + .../GhsaDependencyInjectionRoutine.cs | 53 + .../GhsaServiceCollectionExtensions.cs | 37 + .../Internal/GhsaCursor.cs | 135 + .../Internal/GhsaDiagnostics.cs | 50 + .../Internal/GhsaListParser.cs | 115 + .../Internal/GhsaMapper.cs | 209 + .../Internal/GhsaRecordDto.cs | 55 + .../Internal/GhsaRecordParser.cs | 205 + src/StellaOps.Feedser.Source.Ghsa/Jobs.cs | 43 + .../Properties/AssemblyInfo.cs | 3 + .../StellaOps.Feedser.Source.Ghsa.csproj | 16 + src/StellaOps.Feedser.Source.Ghsa/TASKS.md | 13 + .../AGENTS.md | 39 + .../Class1.cs | 29 + .../StellaOps.Feedser.Source.Ics.Cisa.csproj | 16 + .../TASKS.md | 11 + .../Fixtures/detail-acme-controller-2024.html | 18 + .../Kaspersky/Fixtures/expected-advisory.json | 515 +++ .../Kaspersky/Fixtures/feed-page1.xml | 17 + .../Kaspersky/KasperskyConnectorTests.cs | 346 ++ ....Feedser.Source.Ics.Kaspersky.Tests.csproj | 16 + .../AGENTS.md | 28 + .../Configuration/KasperskyOptions.cs | 53 + .../Internal/KasperskyAdvisoryDto.cs | 14 + .../Internal/KasperskyAdvisoryParser.cs | 172 + .../Internal/KasperskyCursor.cs | 207 + .../Internal/KasperskyFeedClient.cs | 133 + .../Internal/KasperskyFeedItem.cs | 9 + .../Jobs.cs | 46 + .../KasperskyConnector.cs | 464 +++ .../KasperskyConnectorPlugin.cs | 19 + .../KasperskyDependencyInjectionRoutine.cs | 54 + .../KasperskyServiceCollectionExtensions.cs | 37 + ...llaOps.Feedser.Source.Ics.Kaspersky.csproj | 16 + .../TASKS.md | 10 + .../Jvn/Fixtures/expected-advisory.json | 87 + .../Jvn/Fixtures/jvnrss-window1.xml | 53 + .../Jvn/Fixtures/vuldef-JVNDB-2024-123456.xml | 101 + .../Jvn/JvnConnectorTests.cs | 311 ++ .../StellaOps.Feedser.Source.Jvn.Tests.csproj | 16 + src/StellaOps.Feedser.Source.Jvn/AGENTS.md | 29 + .../Configuration/JvnOptions.cs | 80 + .../Internal/JvnAdvisoryMapper.cs | 418 ++ .../Internal/JvnConstants.cs | 10 + .../Internal/JvnCursor.cs | 106 + .../Internal/JvnDetailDto.cs | 69 + .../Internal/JvnDetailParser.cs | 268 ++ .../Internal/JvnOverviewItem.cs | 8 + .../Internal/JvnOverviewPage.cs | 7 + .../Internal/JvnSchemaProvider.cs | 167 + .../Internal/JvnSchemaValidationException.cs | 16 + .../Internal/MyJvnClient.cs | 240 ++ src/StellaOps.Feedser.Source.Jvn/Jobs.cs | 46 + .../JvnConnector.cs | 325 ++ .../JvnConnectorPlugin.cs | 19 + .../JvnDependencyInjectionRoutine.cs | 54 + .../JvnServiceCollectionExtensions.cs | 37 + .../Schemas/data_marking.xsd | 91 + .../Schemas/jvnrss_3.2.xsd | 133 + .../Schemas/mod_sec_3.0.xsd | 168 + .../Schemas/status_3.3.xsd | 574 +++ .../Schemas/tlp_marking.xsd | 40 + .../Schemas/vuldef_3.2.xsd | 1566 ++++++++ .../Schemas/xml.xsd | 287 ++ .../StellaOps.Feedser.Source.Jvn.csproj | 15 + src/StellaOps.Feedser.Source.Jvn/TASKS.md | 13 + .../Kev/Fixtures/kev-advisories.snapshot.json | 271 ++ .../Kev/Fixtures/kev-catalog.json | 38 + .../Kev/KevConnectorTests.cs | 218 ++ .../Kev/KevMapperTests.cs | 70 + .../StellaOps.Feedser.Source.Kev.Tests.csproj | 19 + src/StellaOps.Feedser.Source.Kev/AGENTS.md | 44 + .../Configuration/KevOptions.cs | 33 + .../Internal/KevCatalogDto.cs | 59 + .../Internal/KevCursor.cs | 103 + .../Internal/KevDiagnostics.cs | 48 + .../Internal/KevMapper.cs | 324 ++ .../Internal/KevSchemaProvider.cs | 25 + src/StellaOps.Feedser.Source.Kev/Jobs.cs | 46 + .../KevConnector.cs | 323 ++ .../KevConnectorPlugin.cs | 19 + .../KevDependencyInjectionRoutine.cs | 54 + .../KevServiceCollectionExtensions.cs | 37 + .../Schemas/kev-catalog.schema.json | 80 + .../StellaOps.Feedser.Source.Kev.csproj | 23 + src/StellaOps.Feedser.Source.Kev/TASKS.md | 11 + src/StellaOps.Feedser.Source.Kisa/AGENTS.md | 38 + src/StellaOps.Feedser.Source.Kisa/Class1.cs | 29 + .../StellaOps.Feedser.Source.Kisa.csproj | 16 + src/StellaOps.Feedser.Source.Kisa/TASKS.md | 10 + .../Nvd/Fixtures/nvd-invalid-schema.json | 6 + .../Nvd/Fixtures/nvd-multipage-1.json | 69 + .../Nvd/Fixtures/nvd-multipage-2.json | 69 + .../Nvd/Fixtures/nvd-multipage-3.json | 38 + .../Nvd/Fixtures/nvd-window-1.json | 101 + .../Nvd/Fixtures/nvd-window-2.json | 45 + .../Nvd/Fixtures/nvd-window-update.json | 51 + .../Nvd/NvdConnectorHarnessTests.cs | 136 + .../Nvd/NvdConnectorTests.cs | 653 ++++ .../StellaOps.Feedser.Source.Nvd.Tests.csproj | 16 + src/StellaOps.Feedser.Source.Nvd/AGENTS.md | 26 + .../Configuration/NvdOptions.cs | 57 + .../Internal/NvdCursor.cs | 64 + .../Internal/NvdDiagnostics.cs | 76 + .../Internal/NvdMapper.cs | 596 +++ .../Internal/NvdSchemaProvider.cs | 25 + .../NvdConnector.cs | 565 +++ .../NvdConnectorPlugin.cs | 19 + .../NvdServiceCollectionExtensions.cs | 35 + .../Schemas/nvd-vulnerability.schema.json | 115 + .../StellaOps.Feedser.Source.Nvd.csproj | 17 + src/StellaOps.Feedser.Source.Nvd/TASKS.md | 15 + .../Fixtures/osv-ghsa.ghsa.json | 1183 ++++++ .../Fixtures/osv-ghsa.osv.json | 3363 +++++++++++++++++ .../Fixtures/osv-ghsa.raw-ghsa.json | 519 +++ .../Fixtures/osv-ghsa.raw-osv.json | 714 ++++ .../Fixtures/osv-npm.snapshot.json | 123 + .../Fixtures/osv-pypi.snapshot.json | 123 + .../Osv/OsvGhsaParityRegressionTests.cs | 572 +++ .../Osv/OsvMapperTests.cs | 123 + .../Osv/OsvSnapshotTests.cs | 141 + .../StellaOps.Feedser.Source.Osv.Tests.csproj | 18 + src/StellaOps.Feedser.Source.Osv/AGENTS.md | 26 + .../Configuration/OsvOptions.cs | 81 + .../Internal/OsvCursor.cs | 290 ++ .../Internal/OsvMapper.cs | 484 +++ .../Internal/OsvVulnerabilityDto.cs | 129 + src/StellaOps.Feedser.Source.Osv/Jobs.cs | 46 + .../OsvConnector.cs | 500 +++ .../OsvConnectorPlugin.cs | 20 + .../OsvDependencyInjectionRoutine.cs | 53 + .../OsvServiceCollectionExtensions.cs | 37 + .../Properties/AssemblyInfo.cs | 3 + .../StellaOps.Feedser.Source.Osv.csproj | 23 + src/StellaOps.Feedser.Source.Osv/TASKS.md | 15 + src/StellaOps.Feedser.Source.Ru.Bdu/AGENTS.md | 38 + src/StellaOps.Feedser.Source.Ru.Bdu/Class1.cs | 29 + .../StellaOps.Feedser.Source.Ru.Bdu.csproj | 16 + src/StellaOps.Feedser.Source.Ru.Bdu/TASKS.md | 11 + .../AGENTS.md | 38 + .../Class1.cs | 29 + .../StellaOps.Feedser.Source.Ru.Nkcki.csproj | 16 + .../TASKS.md | 11 + .../Adobe/AdobeConnectorFetchTests.cs | 440 +++ .../Fixtures/adobe-advisories.snapshot.json | 466 +++ .../Fixtures/adobe-detail-apsb25-85.html | 72 + .../Fixtures/adobe-detail-apsb25-87.html | 52 + .../Adobe/Fixtures/adobe-index.html | 17 + ...Ops.Feedser.Source.Vndr.Adobe.Tests.csproj | 17 + .../AGENTS.md | 28 + .../AdobeConnector.cs | 720 ++++ .../AdobeConnectorPlugin.cs | 21 + .../AdobeDiagnostics.cs | 49 + .../AdobeServiceCollectionExtensions.cs | 38 + .../Configuration/AdobeOptions.cs | 50 + .../Internal/AdobeBulletinDto.cs | 102 + .../Internal/AdobeCursor.cs | 168 + .../Internal/AdobeDetailParser.cs | 405 ++ .../Internal/AdobeDocumentMetadata.cs | 47 + .../Internal/AdobeIndexEntry.cs | 5 + .../Internal/AdobeIndexParser.cs | 159 + .../Internal/AdobeSchemaProvider.cs | 25 + .../Schemas/adobe-bulletin.schema.json | 78 + ...StellaOps.Feedser.Source.Vndr.Adobe.csproj | 25 + .../TASKS.md | 11 + .../Apple/AppleConnectorTests.cs | 152 + .../Apple/AppleFixtureManager.cs | 288 ++ .../Apple/AppleLiveRegressionTests.cs | 60 + .../Apple/Fixtures/106355.expected.json | 151 + .../Apple/Fixtures/106355.html | 58 + .../Apple/Fixtures/125326.expected.json | 157 + .../Apple/Fixtures/125326.html | 63 + .../Apple/Fixtures/125328.expected.json | 151 + .../Apple/Fixtures/125328.html | 58 + .../Apple/Fixtures/ht214108.html | 52 + .../Apple/Fixtures/ht215500.html | 47 + .../Apple/Fixtures/index.json | 36 + ...Ops.Feedser.Source.Vndr.Apple.Tests.csproj | 18 + .../AGENTS.md | 39 + .../AppleConnector.cs | 439 +++ .../AppleDependencyInjectionRoutine.cs | 53 + .../AppleOptions.cs | 101 + .../AppleServiceCollectionExtensions.cs | 44 + .../Internal/AppleCursor.cs | 114 + .../Internal/AppleDetailDto.cs | 50 + .../Internal/AppleDetailParser.cs | 326 ++ .../Internal/AppleDiagnostics.cs | 62 + .../Internal/AppleIndexEntry.cs | 144 + .../Internal/AppleMapper.cs | 244 ++ .../Jobs.cs | 46 + .../Properties/AssemblyInfo.cs | 3 + .../README.md | 40 + ...StellaOps.Feedser.Source.Vndr.Apple.csproj | 18 + .../TASKS.md | 11 + .../VndrAppleConnectorPlugin.cs | 24 + .../Chromium/ChromiumConnectorTests.cs | 360 ++ .../Chromium/ChromiumMapperTests.cs | 47 + .../Fixtures/chromium-advisory.snapshot.json | 1 + .../Chromium/Fixtures/chromium-detail.html | 21 + .../Chromium/Fixtures/chromium-feed.xml | 16 + ....Feedser.Source.Vndr.Chromium.Tests.csproj | 18 + .../AGENTS.md | 28 + .../ChromiumConnector.cs | 366 ++ .../ChromiumConnectorPlugin.cs | 20 + .../ChromiumDiagnostics.cs | 69 + .../ChromiumServiceCollectionExtensions.cs | 37 + .../Configuration/ChromiumOptions.cs | 44 + .../Internal/ChromiumCursor.cs | 143 + .../Internal/ChromiumDocumentMetadata.cs | 78 + .../Internal/ChromiumDto.cs | 39 + .../Internal/ChromiumFeedEntry.cs | 24 + .../Internal/ChromiumFeedLoader.cs | 147 + .../Internal/ChromiumMapper.cs | 174 + .../Internal/ChromiumParser.cs | 282 ++ .../Internal/ChromiumSchemaProvider.cs | 25 + .../Properties/AssemblyInfo.cs | 3 + .../Schemas/chromium-post.schema.json | 97 + ...llaOps.Feedser.Source.Vndr.Chromium.csproj | 32 + .../TASKS.md | 17 + .../AGENTS.md | 30 + .../Class1.cs | 29 + ...StellaOps.Feedser.Source.Vndr.Cisco.csproj | 16 + .../TASKS.md | 10 + .../AGENTS.md | 30 + .../Class1.cs | 29 + .../StellaOps.Feedser.Source.Vndr.Msrc.csproj | 16 + .../TASKS.md | 10 + .../Fixtures/oracle-advisories.snapshot.json | 495 +++ .../oracle-calendar-cpuapr2024-single.html | 7 + .../Fixtures/oracle-calendar-cpuapr2024.html | 8 + .../Fixtures/oracle-detail-cpuapr2024-01.html | 108 + .../Fixtures/oracle-detail-cpuapr2024-02.html | 105 + .../Fixtures/oracle-detail-invalid.html | 4 + .../Oracle/OracleConnectorTests.cs | 353 ++ ...ps.Feedser.Source.Vndr.Oracle.Tests.csproj | 17 + .../AGENTS.md | 27 + .../Configuration/OracleOptions.cs | 39 + .../Internal/OracleAffectedEntry.cs | 10 + .../Internal/OracleCalendarFetcher.cs | 92 + .../Internal/OracleCursor.cs | 227 ++ .../Internal/OracleDocumentMetadata.cs | 56 + .../Internal/OracleDto.cs | 16 + .../Internal/OracleDtoValidator.cs | 276 ++ .../Internal/OracleMapper.cs | 426 +++ .../Internal/OracleParser.cs | 457 +++ .../Internal/OraclePatchDocument.cs | 8 + .../Jobs.cs | 46 + .../OracleConnector.cs | 366 ++ .../OracleDependencyInjectionRoutine.cs | 54 + .../OracleServiceCollectionExtensions.cs | 42 + .../Properties/AssemblyInfo.cs | 3 + ...tellaOps.Feedser.Source.Vndr.Oracle.csproj | 17 + .../TASKS.md | 13 + .../VndrOracleConnectorPlugin.cs | 21 + ...ps.Feedser.Source.Vndr.Vmware.Tests.csproj | 18 + .../Fixtures/vmware-advisories.snapshot.json | 275 ++ .../vmware-detail-vmsa-2024-0001.json | 33 + .../vmware-detail-vmsa-2024-0002.json | 27 + .../vmware-detail-vmsa-2024-0003.json | 23 + .../Vmware/Fixtures/vmware-index-initial.json | 12 + .../Vmware/Fixtures/vmware-index-second.json | 17 + .../Vmware/VmwareConnectorTests.cs | 266 ++ .../Vmware/VmwareMapperTests.cs | 86 + .../AGENTS.md | 28 + .../Configuration/VmwareOptions.cs | 54 + .../Internal/VmwareCursor.cs | 172 + .../Internal/VmwareDetailDto.cs | 53 + .../Internal/VmwareFetchCacheEntry.cs | 88 + .../Internal/VmwareIndexItem.cs | 16 + .../Internal/VmwareMapper.cs | 235 ++ .../Jobs.cs | 46 + .../Properties/AssemblyInfo.cs | 3 + ...tellaOps.Feedser.Source.Vndr.Vmware.csproj | 23 + .../TASKS.md | 17 + .../VmwareConnector.cs | 454 +++ .../VmwareConnectorPlugin.cs | 20 + .../VmwareDependencyInjectionRoutine.cs | 53 + .../VmwareDiagnostics.cs | 67 + .../VmwareServiceCollectionExtensions.cs | 39 + .../AdvisoryStorePerformanceTests.cs | 195 + .../AdvisoryStoreTests.cs | 162 + .../AliasStoreTests.cs | 60 + .../DocumentStoreTests.cs | 51 + .../DtoStoreTests.cs | 40 + .../ExportStateManagerTests.cs | 208 + .../ExportStateStoreTests.cs | 42 + .../MergeEventStoreTests.cs | 35 + .../Migrations/MongoMigrationRunnerTests.cs | 238 ++ .../MongoJobStoreTests.cs | 113 + .../MongoSourceStateRepositoryTests.cs | 55 + .../RawDocumentRetentionServiceTests.cs | 93 + ...ellaOps.Feedser.Storage.Mongo.Tests.csproj | 12 + src/StellaOps.Feedser.Storage.Mongo/AGENTS.md | 29 + .../Advisories/AdvisoryDocument.cs | 32 + .../Advisories/AdvisoryStore.cs | 461 +++ .../Advisories/IAdvisoryStore.cs | 14 + .../Advisories/NormalizedVersionDocument.cs | 64 + .../NormalizedVersionDocumentFactory.cs | 100 + .../Aliases/AliasDocument.cs | 38 + .../Aliases/AliasStore.cs | 157 + .../Aliases/AliasStoreConstants.cs | 7 + .../Aliases/AliasStoreMetrics.cs | 22 + .../Aliases/IAliasStore.cs | 27 + .../ChangeHistory/ChangeHistoryDocument.cs | 43 + .../ChangeHistoryDocumentExtensions.cs | 70 + .../ChangeHistory/ChangeHistoryFieldChange.cs | 24 + .../ChangeHistory/ChangeHistoryRecord.cs | 62 + .../ChangeHistory/IChangeHistoryStore.cs | 12 + .../ChangeHistory/MongoChangeHistoryStore.cs | 53 + .../Documents/DocumentDocument.cs | 131 + .../Documents/DocumentRecord.cs | 22 + .../Documents/DocumentStore.cs | 68 + .../Documents/IDocumentStore.cs | 12 + .../Dtos/DtoDocument.cs | 50 + .../Dtos/DtoRecord.cs | 11 + .../Dtos/DtoStore.cs | 57 + .../Dtos/IDtoStore.cs | 10 + .../Exporting/ExportStateDocument.cs | 90 + .../Exporting/ExportStateManager.cs | 135 + .../Exporting/ExportStateRecord.cs | 15 + .../Exporting/ExportStateStore.cs | 43 + .../Exporting/IExportStateStore.cs | 8 + .../ISourceStateRepository.cs | 14 + .../JobLeaseDocument.cs | 38 + .../JobRunDocument.cs | 119 + .../JpFlags/IJpFlagStore.cs | 11 + .../JpFlags/JpFlagDocument.cs | 54 + .../JpFlags/JpFlagRecord.cs | 15 + .../JpFlags/JpFlagStore.cs | 39 + .../MIGRATIONS.md | 37 + .../MergeEvents/IMergeEventStore.cs | 8 + .../MergeEvents/MergeEventDocument.cs | 101 + .../MergeEvents/MergeEventRecord.cs | 10 + .../MergeEvents/MergeEventStore.cs | 36 + .../MergeEvents/MergeFieldDecision.cs | 8 + .../EnsureDocumentExpiryIndexesMigration.cs | 146 + .../EnsureGridFsExpiryIndexesMigration.cs | 95 + .../Migrations/IMongoMigration.cs | 24 + .../Migrations/MongoMigrationDocument.cs | 18 + .../Migrations/MongoMigrationRunner.cs | 102 + .../SemVerStyleBackfillMigration.cs | 81 + .../MongoBootstrapper.cs | 308 ++ .../MongoJobStore.cs | 194 + .../MongoLeaseStore.cs | 116 + .../MongoSourceStateRepository.cs | 112 + .../MongoStorageDefaults.cs | 28 + .../MongoStorageOptions.cs | 93 + .../Properties/AssemblyInfo.cs | 3 + .../PsirtFlags/IPsirtFlagStore.cs | 11 + .../PsirtFlags/PsirtFlagDocument.cs | 52 + .../PsirtFlags/PsirtFlagRecord.cs | 15 + .../PsirtFlags/PsirtFlagStore.cs | 50 + .../RawDocumentRetentionService.cs | 155 + .../ServiceCollectionExtensions.cs | 90 + .../SourceStateDocument.cs | 73 + .../SourceStateRecord.cs | 15 + .../SourceStateRepositoryExtensions.cs | 19 + .../StellaOps.Feedser.Storage.Mongo.csproj | 19 + src/StellaOps.Feedser.Storage.Mongo/TASKS.md | 19 + .../ConnectorTestHarness.cs | 118 + .../MongoIntegrationFixture.cs | 27 + .../StellaOps.Feedser.Testing.csproj | 20 + .../AssemblyInfo.cs | 3 + .../MongoFixtureCollection.cs | 6 + .../FeedserOptionsPostConfigureTests.cs | 56 + .../PluginLoaderTests.cs | 29 + .../StellaOps.Feedser.WebService.Tests.csproj | 13 + .../WebServiceEndpointsTests.cs | 652 ++++ src/StellaOps.Feedser.WebService/AGENTS.md | 34 + .../Diagnostics/HealthContracts.cs | 32 + .../Diagnostics/JobMetrics.cs | 25 + .../Diagnostics/ProblemTypes.cs | 12 + .../Diagnostics/ServiceStatus.cs | 74 + .../Extensions/ConfigurationExtensions.cs | 38 + .../Extensions/JobRegistrationExtensions.cs | 98 + .../Extensions/TelemetryExtensions.cs | 217 ++ .../Filters/JobAuthorizationAuditFilter.cs | 104 + .../Jobs/JobDefinitionResponse.cs | 23 + .../Jobs/JobRunResponse.cs | 29 + .../Jobs/JobTriggerRequest.cs | 8 + .../Options/FeedserOptions.cs | 88 + .../Options/FeedserOptionsPostConfigure.cs | 46 + .../Options/FeedserOptionsValidator.cs | 166 + src/StellaOps.Feedser.WebService/Program.cs | 672 ++++ .../Properties/launchSettings.json | 12 + .../StellaOps.Feedser.WebService.csproj | 35 + src/StellaOps.Feedser.WebService/TASKS.md | 24 + src/StellaOps.Feedser.sln | 916 +++++ .../PluginDependencyInjectionExtensions.cs | 91 + .../StellaOpsPluginRegistration.cs | 26 + .../Hosting/PluginAssembly.cs | 21 + src/StellaOps.Plugin/Hosting/PluginHost.cs | 216 ++ .../Hosting/PluginHostOptions.cs | 59 + .../Hosting/PluginHostResult.cs | 26 + .../Hosting/PluginLoadContext.cs | 79 + .../Internal/ReflectionExtensions.cs | 21 + src/StellaOps.Plugin/PluginContracts.cs | 172 + src/StellaOps.Plugin/StellaOps.Plugin.csproj | 19 + src/StellaOps.Web/AGENTS.md | 24 + src/StellaOps.Web/TASKS.md | 5 + src/StellaOps.sln | 1302 +++++++ src/farewell.txt | 1 + src/global.json | 6 + tools/FixtureUpdater/FixtureUpdater.csproj | 19 + tools/FixtureUpdater/Program.cs | 231 ++ 1132 files changed, 117842 insertions(+) create mode 100644 .gitea/workflows/_deprecated-feedser-ci.yml.disabled create mode 100644 .gitea/workflows/_deprecated-feedser-tests.yml.disabled create mode 100644 .gitea/workflows/build-test-deploy.yml create mode 100755 .gitea/workflows/docs.yml create mode 100644 .gitea/workflows/promote.yml create mode 100644 .gitignore create mode 100644 AGENTS.md create mode 100755 LICENSE create mode 100755 README.md create mode 100644 SPRINTS.md create mode 100644 TODOS.md create mode 100644 certificates/russian_trusted_sub_ca.cer create mode 100755 docs/01_WHAT_IS_IT.md create mode 100755 docs/02_WHY.md create mode 100755 docs/03_QUICKSTART.md create mode 100755 docs/03_VISION.md create mode 100755 docs/04_FEATURE_MATRIX.md create mode 100755 docs/05_ROADMAP.md create mode 100755 docs/05_SYSTEM_REQUIREMENTS_SPEC.md create mode 100755 docs/07_HIGH_LEVEL_ARCHITECTURE.md create mode 100755 docs/08_MODULE_SPECIFICATIONS.md create mode 100755 docs/09_API_CLI_REFERENCE.md create mode 100644 docs/10_FEEDSER_CLI_QUICKSTART.md create mode 100755 docs/10_OFFLINE_KIT.md create mode 100755 docs/10_PLUGIN_SDK_GUIDE.md create mode 100755 docs/11_DATA_SCHEMAS.md create mode 100755 docs/11_GOVERNANCE.md create mode 100755 docs/12_CODE_OF_CONDUCT.md create mode 100755 docs/12_PERFORMANCE_WORKBOOK.md create mode 100755 docs/13_RELEASE_ENGINEERING_PLAYBOOK.md create mode 100755 docs/13_SECURITY_POLICY.md create mode 100755 docs/14_GLOSSARY_OF_TERMS.md create mode 100755 docs/15_UI_GUIDE.md create mode 100755 docs/17_SECURITY_HARDENING_GUIDE.md create mode 100755 docs/18_CODING_STANDARDS.md create mode 100755 docs/19_TEST_SUITE_OVERVIEW.md create mode 100755 docs/21_INSTALL_GUIDE.md create mode 100755 docs/23_FAQ_MATRIX.md create mode 100755 docs/24_OFFLINE_KIT.md create mode 100755 docs/29_LEGAL_FAQ_QUOTA.md create mode 100755 docs/30_QUOTA_ENFORCEMENT_FLOW1.md create mode 100755 docs/33_333_QUOTA_OVERVIEW.md create mode 100755 docs/40_ARCHITECTURE_OVERVIEW.md create mode 100755 docs/60_POLICY_TEMPLATES.md create mode 100644 docs/AGENTS.md create mode 100644 docs/ARCHITECTURE_FEEDSER.md create mode 100755 docs/README.md create mode 100644 docs/TASKS.md create mode 100755 docs/_includes/CONSTANTS.md create mode 100755 docs/ci/20_CI_RECIPES.md create mode 100755 docs/cli/20_REFERENCE.md create mode 100755 docs/dev/30_PLUGIN_DEV_GUIDE.md create mode 100644 docs/dev/31_AUTHORITY_PLUGIN_DEVELOPER_GUIDE.md create mode 100644 docs/dev/32_AUTH_CLIENT_GUIDE.md create mode 100644 docs/dev/authority-rate-limit-tuning-outline.md create mode 100644 docs/dev/merge_semver_playbook.md create mode 100644 docs/dev/mongo_indices.md create mode 100755 docs/license-jwt-quota.md create mode 100644 docs/ops/authority-backup-restore.md create mode 100644 docs/ops/authority-grafana-dashboard.json create mode 100644 docs/ops/authority-monitoring.md create mode 100644 docs/ops/feedser-conflict-resolution.md create mode 100644 docs/ops/feedser-cve-kev-operations.md create mode 100644 docs/ops/migrations/SEMVER_STYLE.md create mode 100644 docs/rfcs/authority-plugin-ldap.md create mode 100644 etc/authority.plugins/ldap.yaml create mode 100644 etc/authority.plugins/standard.yaml create mode 100644 etc/authority.yaml.sample create mode 100644 etc/feedser.yaml.sample create mode 100644 ops/authority/AGENTS.md create mode 100644 ops/authority/Dockerfile create mode 100644 ops/authority/README.md create mode 100644 ops/authority/TASKS.md create mode 100644 ops/authority/docker-compose.authority.yaml create mode 100644 scripts/render_docs.py create mode 100644 scripts/update-model-goldens.ps1 create mode 100644 scripts/update-model-goldens.sh create mode 100644 src/DEDUP_CONFLICTS_RESOLUTION_ALGO.md create mode 100644 src/Directory.Build.props create mode 100644 src/Directory.Build.targets create mode 100644 src/FASTER_MODELING_AND_NORMALIZATION.md create mode 100644 src/StellaOps.Authority/AGENTS.md create mode 100644 src/StellaOps.Authority/StellaOps.Auth.Abstractions.Tests/NetworkMaskMatcherTests.cs create mode 100644 src/StellaOps.Authority/StellaOps.Auth.Abstractions.Tests/StellaOps.Auth.Abstractions.Tests.csproj create mode 100644 src/StellaOps.Authority/StellaOps.Auth.Abstractions.Tests/StellaOpsPrincipalBuilderTests.cs create mode 100644 src/StellaOps.Authority/StellaOps.Auth.Abstractions.Tests/StellaOpsProblemResultFactoryTests.cs create mode 100644 src/StellaOps.Authority/StellaOps.Auth.Abstractions/AuthorityTelemetry.cs create mode 100644 src/StellaOps.Authority/StellaOps.Auth.Abstractions/NetworkMask.cs create mode 100644 src/StellaOps.Authority/StellaOps.Auth.Abstractions/NetworkMaskMatcher.cs create mode 100644 src/StellaOps.Authority/StellaOps.Auth.Abstractions/README.NuGet.md create mode 100644 src/StellaOps.Authority/StellaOps.Auth.Abstractions/StellaOps.Auth.Abstractions.csproj create mode 100644 src/StellaOps.Authority/StellaOps.Auth.Abstractions/StellaOpsAuthenticationDefaults.cs create mode 100644 src/StellaOps.Authority/StellaOps.Auth.Abstractions/StellaOpsClaimTypes.cs create mode 100644 src/StellaOps.Authority/StellaOps.Auth.Abstractions/StellaOpsPrincipalBuilder.cs create mode 100644 src/StellaOps.Authority/StellaOps.Auth.Abstractions/StellaOpsProblemResultFactory.cs create mode 100644 src/StellaOps.Authority/StellaOps.Auth.Abstractions/StellaOpsScopes.cs create mode 100644 src/StellaOps.Authority/StellaOps.Auth.Client.Tests/ServiceCollectionExtensionsTests.cs create mode 100644 src/StellaOps.Authority/StellaOps.Auth.Client.Tests/StellaOps.Auth.Client.Tests.csproj create mode 100644 src/StellaOps.Authority/StellaOps.Auth.Client.Tests/StellaOpsAuthClientOptionsTests.cs create mode 100644 src/StellaOps.Authority/StellaOps.Auth.Client.Tests/StellaOpsDiscoveryCacheTests.cs create mode 100644 src/StellaOps.Authority/StellaOps.Auth.Client.Tests/StellaOpsTokenClientTests.cs create mode 100644 src/StellaOps.Authority/StellaOps.Auth.Client.Tests/TokenCacheTests.cs create mode 100644 src/StellaOps.Authority/StellaOps.Auth.Client/FileTokenCache.cs create mode 100644 src/StellaOps.Authority/StellaOps.Auth.Client/IStellaOpsTokenCache.cs create mode 100644 src/StellaOps.Authority/StellaOps.Auth.Client/IStellaOpsTokenClient.cs create mode 100644 src/StellaOps.Authority/StellaOps.Auth.Client/InMemoryTokenCache.cs create mode 100644 src/StellaOps.Authority/StellaOps.Auth.Client/README.NuGet.md create mode 100644 src/StellaOps.Authority/StellaOps.Auth.Client/ServiceCollectionExtensions.cs create mode 100644 src/StellaOps.Authority/StellaOps.Auth.Client/StellaOps.Auth.Client.csproj create mode 100644 src/StellaOps.Authority/StellaOps.Auth.Client/StellaOpsAuthClientOptions.cs create mode 100644 src/StellaOps.Authority/StellaOps.Auth.Client/StellaOpsDiscoveryCache.cs create mode 100644 src/StellaOps.Authority/StellaOps.Auth.Client/StellaOpsJwksCache.cs create mode 100644 src/StellaOps.Authority/StellaOps.Auth.Client/StellaOpsTokenCacheEntry.cs create mode 100644 src/StellaOps.Authority/StellaOps.Auth.Client/StellaOpsTokenClient.cs create mode 100644 src/StellaOps.Authority/StellaOps.Auth.Client/StellaOpsTokenResult.cs create mode 100644 src/StellaOps.Authority/StellaOps.Auth.ServerIntegration.Tests/ServiceCollectionExtensionsTests.cs create mode 100644 src/StellaOps.Authority/StellaOps.Auth.ServerIntegration.Tests/StellaOps.Auth.ServerIntegration.Tests.csproj create mode 100644 src/StellaOps.Authority/StellaOps.Auth.ServerIntegration.Tests/StellaOpsResourceServerOptionsTests.cs create mode 100644 src/StellaOps.Authority/StellaOps.Auth.ServerIntegration.Tests/StellaOpsScopeAuthorizationHandlerTests.cs create mode 100644 src/StellaOps.Authority/StellaOps.Auth.ServerIntegration/README.NuGet.md create mode 100644 src/StellaOps.Authority/StellaOps.Auth.ServerIntegration/ServiceCollectionExtensions.cs create mode 100644 src/StellaOps.Authority/StellaOps.Auth.ServerIntegration/StellaOps.Auth.ServerIntegration.csproj create mode 100644 src/StellaOps.Authority/StellaOps.Auth.ServerIntegration/StellaOpsAuthorizationPolicyBuilderExtensions.cs create mode 100644 src/StellaOps.Authority/StellaOps.Auth.ServerIntegration/StellaOpsBypassEvaluator.cs create mode 100644 src/StellaOps.Authority/StellaOps.Auth.ServerIntegration/StellaOpsResourceServerOptions.cs create mode 100644 src/StellaOps.Authority/StellaOps.Auth.ServerIntegration/StellaOpsScopeAuthorizationHandler.cs create mode 100644 src/StellaOps.Authority/StellaOps.Auth.ServerIntegration/StellaOpsScopeRequirement.cs create mode 100644 src/StellaOps.Authority/StellaOps.Authority.Plugin.Standard.Tests/StandardClientProvisioningStoreTests.cs create mode 100644 src/StellaOps.Authority/StellaOps.Authority.Plugin.Standard.Tests/StandardPluginOptionsTests.cs create mode 100644 src/StellaOps.Authority/StellaOps.Authority.Plugin.Standard.Tests/StandardPluginRegistrarTests.cs create mode 100644 src/StellaOps.Authority/StellaOps.Authority.Plugin.Standard.Tests/StandardUserCredentialStoreTests.cs create mode 100644 src/StellaOps.Authority/StellaOps.Authority.Plugin.Standard.Tests/StellaOps.Authority.Plugin.Standard.Tests.csproj create mode 100644 src/StellaOps.Authority/StellaOps.Authority.Plugin.Standard/AGENTS.md create mode 100644 src/StellaOps.Authority/StellaOps.Authority.Plugin.Standard/Bootstrap/StandardPluginBootstrapper.cs create mode 100644 src/StellaOps.Authority/StellaOps.Authority.Plugin.Standard/Properties/AssemblyInfo.cs create mode 100644 src/StellaOps.Authority/StellaOps.Authority.Plugin.Standard/Security/IPasswordHasher.cs create mode 100644 src/StellaOps.Authority/StellaOps.Authority.Plugin.Standard/StandardClaimsEnricher.cs create mode 100644 src/StellaOps.Authority/StellaOps.Authority.Plugin.Standard/StandardIdentityProviderPlugin.cs create mode 100644 src/StellaOps.Authority/StellaOps.Authority.Plugin.Standard/StandardPluginOptions.cs create mode 100644 src/StellaOps.Authority/StellaOps.Authority.Plugin.Standard/StandardPluginRegistrar.cs create mode 100644 src/StellaOps.Authority/StellaOps.Authority.Plugin.Standard/StellaOps.Authority.Plugin.Standard.csproj create mode 100644 src/StellaOps.Authority/StellaOps.Authority.Plugin.Standard/Storage/StandardClientProvisioningStore.cs create mode 100644 src/StellaOps.Authority/StellaOps.Authority.Plugin.Standard/Storage/StandardUserCredentialStore.cs create mode 100644 src/StellaOps.Authority/StellaOps.Authority.Plugin.Standard/Storage/StandardUserDocument.cs create mode 100644 src/StellaOps.Authority/StellaOps.Authority.Plugin.Standard/TASKS.md create mode 100644 src/StellaOps.Authority/StellaOps.Authority.Plugins.Abstractions.Tests/AuthorityClientRegistrationTests.cs create mode 100644 src/StellaOps.Authority/StellaOps.Authority.Plugins.Abstractions.Tests/AuthorityCredentialVerificationResultTests.cs create mode 100644 src/StellaOps.Authority/StellaOps.Authority.Plugins.Abstractions.Tests/AuthorityIdentityProviderCapabilitiesTests.cs create mode 100644 src/StellaOps.Authority/StellaOps.Authority.Plugins.Abstractions.Tests/AuthorityPluginHealthResultTests.cs create mode 100644 src/StellaOps.Authority/StellaOps.Authority.Plugins.Abstractions.Tests/AuthorityPluginOperationResultTests.cs create mode 100644 src/StellaOps.Authority/StellaOps.Authority.Plugins.Abstractions.Tests/AuthorityUserDescriptorTests.cs create mode 100644 src/StellaOps.Authority/StellaOps.Authority.Plugins.Abstractions.Tests/AuthorityUserRegistrationTests.cs create mode 100644 src/StellaOps.Authority/StellaOps.Authority.Plugins.Abstractions.Tests/StellaOps.Authority.Plugins.Abstractions.Tests.csproj create mode 100644 src/StellaOps.Authority/StellaOps.Authority.Plugins.Abstractions/AuthorityClientMetadataKeys.cs create mode 100644 src/StellaOps.Authority/StellaOps.Authority.Plugins.Abstractions/AuthorityPluginContracts.cs create mode 100644 src/StellaOps.Authority/StellaOps.Authority.Plugins.Abstractions/AuthorityPluginRegistrationContext.cs create mode 100644 src/StellaOps.Authority/StellaOps.Authority.Plugins.Abstractions/AuthoritySecretHasher.cs create mode 100644 src/StellaOps.Authority/StellaOps.Authority.Plugins.Abstractions/IdentityProviderContracts.cs create mode 100644 src/StellaOps.Authority/StellaOps.Authority.Plugins.Abstractions/StellaOps.Authority.Plugins.Abstractions.csproj create mode 100644 src/StellaOps.Authority/StellaOps.Authority.Storage.Mongo/AuthorityMongoDefaults.cs create mode 100644 src/StellaOps.Authority/StellaOps.Authority.Storage.Mongo/Class1.cs create mode 100644 src/StellaOps.Authority/StellaOps.Authority.Storage.Mongo/Documents/AuthorityClientDocument.cs create mode 100644 src/StellaOps.Authority/StellaOps.Authority.Storage.Mongo/Documents/AuthorityLoginAttemptDocument.cs create mode 100644 src/StellaOps.Authority/StellaOps.Authority.Storage.Mongo/Documents/AuthorityScopeDocument.cs create mode 100644 src/StellaOps.Authority/StellaOps.Authority.Storage.Mongo/Documents/AuthorityTokenDocument.cs create mode 100644 src/StellaOps.Authority/StellaOps.Authority.Storage.Mongo/Documents/AuthorityUserDocument.cs create mode 100644 src/StellaOps.Authority/StellaOps.Authority.Storage.Mongo/Extensions/ServiceCollectionExtensions.cs create mode 100644 src/StellaOps.Authority/StellaOps.Authority.Storage.Mongo/Initialization/AuthorityClientCollectionInitializer.cs create mode 100644 src/StellaOps.Authority/StellaOps.Authority.Storage.Mongo/Initialization/AuthorityLoginAttemptCollectionInitializer.cs create mode 100644 src/StellaOps.Authority/StellaOps.Authority.Storage.Mongo/Initialization/AuthorityMongoInitializer.cs create mode 100644 src/StellaOps.Authority/StellaOps.Authority.Storage.Mongo/Initialization/AuthorityScopeCollectionInitializer.cs create mode 100644 src/StellaOps.Authority/StellaOps.Authority.Storage.Mongo/Initialization/AuthorityTokenCollectionInitializer.cs create mode 100644 src/StellaOps.Authority/StellaOps.Authority.Storage.Mongo/Initialization/AuthorityUserCollectionInitializer.cs create mode 100644 src/StellaOps.Authority/StellaOps.Authority.Storage.Mongo/Initialization/IAuthorityCollectionInitializer.cs create mode 100644 src/StellaOps.Authority/StellaOps.Authority.Storage.Mongo/Migrations/AuthorityMongoMigrationRunner.cs create mode 100644 src/StellaOps.Authority/StellaOps.Authority.Storage.Mongo/Migrations/EnsureAuthorityCollectionsMigration.cs create mode 100644 src/StellaOps.Authority/StellaOps.Authority.Storage.Mongo/Migrations/IAuthorityMongoMigration.cs create mode 100644 src/StellaOps.Authority/StellaOps.Authority.Storage.Mongo/Options/AuthorityMongoOptions.cs create mode 100644 src/StellaOps.Authority/StellaOps.Authority.Storage.Mongo/StellaOps.Authority.Storage.Mongo.csproj create mode 100644 src/StellaOps.Authority/StellaOps.Authority.Storage.Mongo/Stores/AuthorityClientStore.cs create mode 100644 src/StellaOps.Authority/StellaOps.Authority.Storage.Mongo/Stores/AuthorityLoginAttemptStore.cs create mode 100644 src/StellaOps.Authority/StellaOps.Authority.Storage.Mongo/Stores/AuthorityScopeStore.cs create mode 100644 src/StellaOps.Authority/StellaOps.Authority.Storage.Mongo/Stores/AuthorityTokenStore.cs create mode 100644 src/StellaOps.Authority/StellaOps.Authority.Storage.Mongo/Stores/AuthorityUserStore.cs create mode 100644 src/StellaOps.Authority/StellaOps.Authority.Storage.Mongo/Stores/IAuthorityClientStore.cs create mode 100644 src/StellaOps.Authority/StellaOps.Authority.Storage.Mongo/Stores/IAuthorityLoginAttemptStore.cs create mode 100644 src/StellaOps.Authority/StellaOps.Authority.Storage.Mongo/Stores/IAuthorityScopeStore.cs create mode 100644 src/StellaOps.Authority/StellaOps.Authority.Storage.Mongo/Stores/IAuthorityTokenStore.cs create mode 100644 src/StellaOps.Authority/StellaOps.Authority.Storage.Mongo/Stores/IAuthorityUserStore.cs create mode 100644 src/StellaOps.Authority/StellaOps.Authority.Tests/Identity/AuthorityIdentityProviderRegistryTests.cs create mode 100644 src/StellaOps.Authority/StellaOps.Authority.Tests/Identity/AuthorityIdentityProviderSelectorTests.cs create mode 100644 src/StellaOps.Authority/StellaOps.Authority.Tests/OpenIddict/ClientCredentialsAndTokenHandlersTests.cs create mode 100644 src/StellaOps.Authority/StellaOps.Authority.Tests/OpenIddict/TokenPersistenceIntegrationTests.cs create mode 100644 src/StellaOps.Authority/StellaOps.Authority.Tests/Plugins/AuthorityPluginLoaderTests.cs create mode 100644 src/StellaOps.Authority/StellaOps.Authority.Tests/RateLimiting/AuthorityRateLimiterIntegrationTests.cs create mode 100644 src/StellaOps.Authority/StellaOps.Authority.Tests/RateLimiting/AuthorityRateLimiterMetadataAccessorTests.cs create mode 100644 src/StellaOps.Authority/StellaOps.Authority.Tests/RateLimiting/AuthorityRateLimiterMetadataMiddlewareTests.cs create mode 100644 src/StellaOps.Authority/StellaOps.Authority.Tests/RateLimiting/AuthorityRateLimiterTests.cs create mode 100644 src/StellaOps.Authority/StellaOps.Authority.Tests/StellaOps.Authority.Tests.csproj create mode 100644 src/StellaOps.Authority/StellaOps.Authority.sln create mode 100644 src/StellaOps.Authority/StellaOps.Authority/AuthorityIdentityProviderRegistry.cs create mode 100644 src/StellaOps.Authority/StellaOps.Authority/AuthorityPluginRegistry.cs create mode 100644 src/StellaOps.Authority/StellaOps.Authority/AuthorityRateLimiter.cs create mode 100644 src/StellaOps.Authority/StellaOps.Authority/AuthorityTelemetryConfiguration.cs create mode 100644 src/StellaOps.Authority/StellaOps.Authority/Bootstrap/BootstrapApiKeyFilter.cs create mode 100644 src/StellaOps.Authority/StellaOps.Authority/Bootstrap/BootstrapRequests.cs create mode 100644 src/StellaOps.Authority/StellaOps.Authority/OpenIddict/AuthorityIdentityProviderSelector.cs create mode 100644 src/StellaOps.Authority/StellaOps.Authority/OpenIddict/AuthorityOpenIddictConstants.cs create mode 100644 src/StellaOps.Authority/StellaOps.Authority/OpenIddict/Handlers/ClientCredentialsHandlers.cs create mode 100644 src/StellaOps.Authority/StellaOps.Authority/OpenIddict/Handlers/PasswordGrantHandlers.cs create mode 100644 src/StellaOps.Authority/StellaOps.Authority/OpenIddict/Handlers/TokenValidationHandlers.cs create mode 100644 src/StellaOps.Authority/StellaOps.Authority/Plugins/AuthorityPluginLoader.cs create mode 100644 src/StellaOps.Authority/StellaOps.Authority/Plugins/AuthorityPluginRegistrationSummary.cs create mode 100644 src/StellaOps.Authority/StellaOps.Authority/Program.cs create mode 100644 src/StellaOps.Authority/StellaOps.Authority/Properties/AssemblyInfo.cs create mode 100644 src/StellaOps.Authority/StellaOps.Authority/Properties/launchSettings.json create mode 100644 src/StellaOps.Authority/StellaOps.Authority/RateLimiting/AuthorityRateLimiterFeature.cs create mode 100644 src/StellaOps.Authority/StellaOps.Authority/RateLimiting/AuthorityRateLimiterMetadata.cs create mode 100644 src/StellaOps.Authority/StellaOps.Authority/RateLimiting/AuthorityRateLimiterMetadataAccessor.cs create mode 100644 src/StellaOps.Authority/StellaOps.Authority/RateLimiting/AuthorityRateLimiterMetadataMiddleware.cs create mode 100644 src/StellaOps.Authority/StellaOps.Authority/RateLimiting/AuthorityRateLimiterPartitionKeyResolver.cs create mode 100644 src/StellaOps.Authority/StellaOps.Authority/RateLimiting/AuthorityRateLimitingApplicationBuilderExtensions.cs create mode 100644 src/StellaOps.Authority/StellaOps.Authority/StellaOps.Authority.csproj create mode 100644 src/StellaOps.Authority/StellaOps.Authority/appsettings.Development.json create mode 100644 src/StellaOps.Authority/StellaOps.Authority/appsettings.json create mode 100644 src/StellaOps.Authority/TASKS.md create mode 100644 src/StellaOps.Cli.Tests/Commands/CommandHandlersTests.cs create mode 100644 src/StellaOps.Cli.Tests/Configuration/CliBootstrapperTests.cs create mode 100644 src/StellaOps.Cli.Tests/Services/BackendOperationsClientTests.cs create mode 100644 src/StellaOps.Cli.Tests/StellaOps.Cli.Tests.csproj create mode 100644 src/StellaOps.Cli.Tests/Testing/TestHelpers.cs create mode 100644 src/StellaOps.Cli.Tests/UnitTest1.cs create mode 100644 src/StellaOps.Cli.Tests/xunit.runner.json create mode 100644 src/StellaOps.Cli/AGENTS.md create mode 100644 src/StellaOps.Cli/Commands/CommandFactory.cs create mode 100644 src/StellaOps.Cli/Commands/CommandHandlers.cs create mode 100644 src/StellaOps.Cli/Configuration/AuthorityTokenUtilities.cs create mode 100644 src/StellaOps.Cli/Configuration/CliBootstrapper.cs create mode 100644 src/StellaOps.Cli/Configuration/StellaOpsCliOptions.cs create mode 100644 src/StellaOps.Cli/Program.cs create mode 100644 src/StellaOps.Cli/Prompts/TrivyDbExportPrompt.cs create mode 100644 src/StellaOps.Cli/Properties/AssemblyInfo.cs create mode 100644 src/StellaOps.Cli/Services/BackendOperationsClient.cs create mode 100644 src/StellaOps.Cli/Services/IBackendOperationsClient.cs create mode 100644 src/StellaOps.Cli/Services/IScannerExecutor.cs create mode 100644 src/StellaOps.Cli/Services/IScannerInstaller.cs create mode 100644 src/StellaOps.Cli/Services/Models/JobTriggerResult.cs create mode 100644 src/StellaOps.Cli/Services/Models/ScannerArtifactResult.cs create mode 100644 src/StellaOps.Cli/Services/Models/Transport/JobRunResponse.cs create mode 100644 src/StellaOps.Cli/Services/Models/Transport/JobTriggerRequest.cs create mode 100644 src/StellaOps.Cli/Services/Models/Transport/ProblemDocument.cs create mode 100644 src/StellaOps.Cli/Services/ScannerExecutionResult.cs create mode 100644 src/StellaOps.Cli/Services/ScannerExecutor.cs create mode 100644 src/StellaOps.Cli/Services/ScannerInstaller.cs create mode 100644 src/StellaOps.Cli/StellaOps.Cli.csproj create mode 100644 src/StellaOps.Cli/TASKS.md create mode 100644 src/StellaOps.Cli/Telemetry/CliActivitySource.cs create mode 100644 src/StellaOps.Cli/Telemetry/CliMetrics.cs create mode 100644 src/StellaOps.Cli/Telemetry/VerbosityState.cs create mode 100644 src/StellaOps.Cli/appsettings.json create mode 100644 src/StellaOps.Configuration.Tests/AuthorityPluginConfigurationLoaderTests.cs create mode 100644 src/StellaOps.Configuration.Tests/AuthorityTelemetryTests.cs create mode 100644 src/StellaOps.Configuration.Tests/StellaOps.Configuration.Tests.csproj create mode 100644 src/StellaOps.Configuration.Tests/StellaOpsAuthorityOptionsTests.cs create mode 100644 src/StellaOps.Configuration/AuthorityPluginConfigurationLoader.cs create mode 100644 src/StellaOps.Configuration/StellaOps.Configuration.csproj create mode 100644 src/StellaOps.Configuration/StellaOpsAuthorityConfiguration.cs create mode 100644 src/StellaOps.Configuration/StellaOpsAuthorityOptions.cs create mode 100644 src/StellaOps.Configuration/StellaOpsBootstrapOptions.cs create mode 100644 src/StellaOps.Configuration/StellaOpsConfigurationBootstrapper.cs create mode 100644 src/StellaOps.Configuration/StellaOpsConfigurationContext.cs create mode 100644 src/StellaOps.Configuration/StellaOpsConfigurationOptions.cs create mode 100644 src/StellaOps.Configuration/StellaOpsOptionsBinder.cs create mode 100644 src/StellaOps.Cryptography.Tests/PasswordHashOptionsTests.cs create mode 100644 src/StellaOps.Cryptography.Tests/StellaOps.Cryptography.Tests.csproj create mode 100644 src/StellaOps.Cryptography/AGENTS.md create mode 100644 src/StellaOps.Cryptography/CryptoProvider.cs create mode 100644 src/StellaOps.Cryptography/PasswordHashing.cs create mode 100644 src/StellaOps.Cryptography/StellaOps.Cryptography.csproj create mode 100644 src/StellaOps.Cryptography/TASKS.md create mode 100644 src/StellaOps.DependencyInjection/IDependencyInjectionRoutine.cs create mode 100644 src/StellaOps.DependencyInjection/StellaOps.DependencyInjection.csproj create mode 100644 src/StellaOps.Feedser.Core.Tests/CanonicalMergerTests.cs create mode 100644 src/StellaOps.Feedser.Core.Tests/JobCoordinatorTests.cs create mode 100644 src/StellaOps.Feedser.Core.Tests/JobPluginRegistrationExtensionsTests.cs create mode 100644 src/StellaOps.Feedser.Core.Tests/JobSchedulerBuilderTests.cs create mode 100644 src/StellaOps.Feedser.Core.Tests/PluginRoutineFixtures.cs create mode 100644 src/StellaOps.Feedser.Core.Tests/StellaOps.Feedser.Core.Tests.csproj create mode 100644 src/StellaOps.Feedser.Core/AGENTS.md create mode 100644 src/StellaOps.Feedser.Core/CanonicalMergeResult.cs create mode 100644 src/StellaOps.Feedser.Core/CanonicalMerger.cs create mode 100644 src/StellaOps.Feedser.Core/Jobs/IJob.cs create mode 100644 src/StellaOps.Feedser.Core/Jobs/IJobCoordinator.cs create mode 100644 src/StellaOps.Feedser.Core/Jobs/IJobStore.cs create mode 100644 src/StellaOps.Feedser.Core/Jobs/ILeaseStore.cs create mode 100644 src/StellaOps.Feedser.Core/Jobs/JobCoordinator.cs create mode 100644 src/StellaOps.Feedser.Core/Jobs/JobDefinition.cs create mode 100644 src/StellaOps.Feedser.Core/Jobs/JobDiagnostics.cs create mode 100644 src/StellaOps.Feedser.Core/Jobs/JobExecutionContext.cs create mode 100644 src/StellaOps.Feedser.Core/Jobs/JobLease.cs create mode 100644 src/StellaOps.Feedser.Core/Jobs/JobPluginRegistrationExtensions.cs create mode 100644 src/StellaOps.Feedser.Core/Jobs/JobRunCompletion.cs create mode 100644 src/StellaOps.Feedser.Core/Jobs/JobRunCreateRequest.cs create mode 100644 src/StellaOps.Feedser.Core/Jobs/JobRunSnapshot.cs create mode 100644 src/StellaOps.Feedser.Core/Jobs/JobRunStatus.cs create mode 100644 src/StellaOps.Feedser.Core/Jobs/JobSchedulerBuilder.cs create mode 100644 src/StellaOps.Feedser.Core/Jobs/JobSchedulerHostedService.cs create mode 100644 src/StellaOps.Feedser.Core/Jobs/JobSchedulerOptions.cs create mode 100644 src/StellaOps.Feedser.Core/Jobs/JobTriggerResult.cs create mode 100644 src/StellaOps.Feedser.Core/Jobs/ServiceCollectionExtensions.cs create mode 100644 src/StellaOps.Feedser.Core/StellaOps.Feedser.Core.csproj create mode 100644 src/StellaOps.Feedser.Core/TASKS.md create mode 100644 src/StellaOps.Feedser.Exporter.Json.Tests/JsonExportSnapshotBuilderTests.cs create mode 100644 src/StellaOps.Feedser.Exporter.Json.Tests/JsonExporterDependencyInjectionRoutineTests.cs create mode 100644 src/StellaOps.Feedser.Exporter.Json.Tests/JsonExporterParitySmokeTests.cs create mode 100644 src/StellaOps.Feedser.Exporter.Json.Tests/JsonFeedExporterTests.cs create mode 100644 src/StellaOps.Feedser.Exporter.Json.Tests/StellaOps.Feedser.Exporter.Json.Tests.csproj create mode 100644 src/StellaOps.Feedser.Exporter.Json.Tests/VulnListJsonExportPathResolverTests.cs create mode 100644 src/StellaOps.Feedser.Exporter.Json/AGENTS.md create mode 100644 src/StellaOps.Feedser.Exporter.Json/ExportDigestCalculator.cs create mode 100644 src/StellaOps.Feedser.Exporter.Json/ExporterVersion.cs create mode 100644 src/StellaOps.Feedser.Exporter.Json/IJsonExportPathResolver.cs create mode 100644 src/StellaOps.Feedser.Exporter.Json/JsonExportFile.cs create mode 100644 src/StellaOps.Feedser.Exporter.Json/JsonExportJob.cs create mode 100644 src/StellaOps.Feedser.Exporter.Json/JsonExportManifestWriter.cs create mode 100644 src/StellaOps.Feedser.Exporter.Json/JsonExportOptions.cs create mode 100644 src/StellaOps.Feedser.Exporter.Json/JsonExportResult.cs create mode 100644 src/StellaOps.Feedser.Exporter.Json/JsonExportSnapshotBuilder.cs create mode 100644 src/StellaOps.Feedser.Exporter.Json/JsonExporterDependencyInjectionRoutine.cs create mode 100644 src/StellaOps.Feedser.Exporter.Json/JsonExporterPlugin.cs create mode 100644 src/StellaOps.Feedser.Exporter.Json/JsonFeedExporter.cs create mode 100644 src/StellaOps.Feedser.Exporter.Json/StellaOps.Feedser.Exporter.Json.csproj create mode 100644 src/StellaOps.Feedser.Exporter.Json/TASKS.md create mode 100644 src/StellaOps.Feedser.Exporter.Json/VulnListJsonExportPathResolver.cs create mode 100644 src/StellaOps.Feedser.Exporter.TrivyDb.Tests/StellaOps.Feedser.Exporter.TrivyDb.Tests.csproj create mode 100644 src/StellaOps.Feedser.Exporter.TrivyDb.Tests/TrivyDbExportPlannerTests.cs create mode 100644 src/StellaOps.Feedser.Exporter.TrivyDb.Tests/TrivyDbFeedExporterTests.cs create mode 100644 src/StellaOps.Feedser.Exporter.TrivyDb.Tests/TrivyDbOciWriterTests.cs create mode 100644 src/StellaOps.Feedser.Exporter.TrivyDb.Tests/TrivyDbPackageBuilderTests.cs create mode 100644 src/StellaOps.Feedser.Exporter.TrivyDb/AGENTS.md create mode 100644 src/StellaOps.Feedser.Exporter.TrivyDb/ITrivyDbBuilder.cs create mode 100644 src/StellaOps.Feedser.Exporter.TrivyDb/ITrivyDbOrasPusher.cs create mode 100644 src/StellaOps.Feedser.Exporter.TrivyDb/OciDescriptor.cs create mode 100644 src/StellaOps.Feedser.Exporter.TrivyDb/OciIndex.cs create mode 100644 src/StellaOps.Feedser.Exporter.TrivyDb/OciManifest.cs create mode 100644 src/StellaOps.Feedser.Exporter.TrivyDb/StellaOps.Feedser.Exporter.TrivyDb.csproj create mode 100644 src/StellaOps.Feedser.Exporter.TrivyDb/TASKS.md create mode 100644 src/StellaOps.Feedser.Exporter.TrivyDb/TrivyConfigDocument.cs create mode 100644 src/StellaOps.Feedser.Exporter.TrivyDb/TrivyDbBlob.cs create mode 100644 src/StellaOps.Feedser.Exporter.TrivyDb/TrivyDbBoltBuilder.cs create mode 100644 src/StellaOps.Feedser.Exporter.TrivyDb/TrivyDbBuilderResult.cs create mode 100644 src/StellaOps.Feedser.Exporter.TrivyDb/TrivyDbExportJob.cs create mode 100644 src/StellaOps.Feedser.Exporter.TrivyDb/TrivyDbExportMode.cs create mode 100644 src/StellaOps.Feedser.Exporter.TrivyDb/TrivyDbExportOptions.cs create mode 100644 src/StellaOps.Feedser.Exporter.TrivyDb/TrivyDbExportOverrides.cs create mode 100644 src/StellaOps.Feedser.Exporter.TrivyDb/TrivyDbExportPlan.cs create mode 100644 src/StellaOps.Feedser.Exporter.TrivyDb/TrivyDbExportPlanner.cs create mode 100644 src/StellaOps.Feedser.Exporter.TrivyDb/TrivyDbExporterDependencyInjectionRoutine.cs create mode 100644 src/StellaOps.Feedser.Exporter.TrivyDb/TrivyDbExporterPlugin.cs create mode 100644 src/StellaOps.Feedser.Exporter.TrivyDb/TrivyDbFeedExporter.cs create mode 100644 src/StellaOps.Feedser.Exporter.TrivyDb/TrivyDbMediaTypes.cs create mode 100644 src/StellaOps.Feedser.Exporter.TrivyDb/TrivyDbOciWriteResult.cs create mode 100644 src/StellaOps.Feedser.Exporter.TrivyDb/TrivyDbOciWriter.cs create mode 100644 src/StellaOps.Feedser.Exporter.TrivyDb/TrivyDbOrasPusher.cs create mode 100644 src/StellaOps.Feedser.Exporter.TrivyDb/TrivyDbPackage.cs create mode 100644 src/StellaOps.Feedser.Exporter.TrivyDb/TrivyDbPackageBuilder.cs create mode 100644 src/StellaOps.Feedser.Exporter.TrivyDb/TrivyDbPackageRequest.cs create mode 100644 src/StellaOps.Feedser.Merge.Tests/AdvisoryIdentityResolverTests.cs create mode 100644 src/StellaOps.Feedser.Merge.Tests/AdvisoryMergeServiceTests.cs create mode 100644 src/StellaOps.Feedser.Merge.Tests/AdvisoryPrecedenceMergerTests.cs create mode 100644 src/StellaOps.Feedser.Merge.Tests/AffectedPackagePrecedenceResolverTests.cs create mode 100644 src/StellaOps.Feedser.Merge.Tests/AliasGraphResolverTests.cs create mode 100644 src/StellaOps.Feedser.Merge.Tests/CanonicalHashCalculatorTests.cs create mode 100644 src/StellaOps.Feedser.Merge.Tests/DebianEvrComparerTests.cs create mode 100644 src/StellaOps.Feedser.Merge.Tests/MergeEventWriterTests.cs create mode 100644 src/StellaOps.Feedser.Merge.Tests/MergePrecedenceIntegrationTests.cs create mode 100644 src/StellaOps.Feedser.Merge.Tests/MetricCollector.cs create mode 100644 src/StellaOps.Feedser.Merge.Tests/NevraComparerTests.cs create mode 100644 src/StellaOps.Feedser.Merge.Tests/SemanticVersionRangeResolverTests.cs create mode 100644 src/StellaOps.Feedser.Merge.Tests/StellaOps.Feedser.Merge.Tests.csproj create mode 100644 src/StellaOps.Feedser.Merge.Tests/TestLogger.cs create mode 100644 src/StellaOps.Feedser.Merge/AGENTS.md create mode 100644 src/StellaOps.Feedser.Merge/Class1.cs create mode 100644 src/StellaOps.Feedser.Merge/Comparers/DebianEvr.cs create mode 100644 src/StellaOps.Feedser.Merge/Comparers/Nevra.cs create mode 100644 src/StellaOps.Feedser.Merge/Comparers/SemanticVersionRangeResolver.cs create mode 100644 src/StellaOps.Feedser.Merge/Identity/AdvisoryIdentityCluster.cs create mode 100644 src/StellaOps.Feedser.Merge/Identity/AdvisoryIdentityResolver.cs create mode 100644 src/StellaOps.Feedser.Merge/Identity/AliasIdentity.cs create mode 100644 src/StellaOps.Feedser.Merge/Jobs/MergeJobKinds.cs create mode 100644 src/StellaOps.Feedser.Merge/Jobs/MergeReconcileJob.cs create mode 100644 src/StellaOps.Feedser.Merge/MergeServiceCollectionExtensions.cs create mode 100644 src/StellaOps.Feedser.Merge/Options/AdvisoryPrecedenceDefaults.cs create mode 100644 src/StellaOps.Feedser.Merge/Options/AdvisoryPrecedenceOptions.cs create mode 100644 src/StellaOps.Feedser.Merge/Options/AdvisoryPrecedenceTable.cs create mode 100644 src/StellaOps.Feedser.Merge/RANGE_PRIMITIVES_COORDINATION.md create mode 100644 src/StellaOps.Feedser.Merge/Services/AdvisoryMergeService.cs create mode 100644 src/StellaOps.Feedser.Merge/Services/AdvisoryPrecedenceMerger.cs create mode 100644 src/StellaOps.Feedser.Merge/Services/AffectedPackagePrecedenceResolver.cs create mode 100644 src/StellaOps.Feedser.Merge/Services/AliasGraphResolver.cs create mode 100644 src/StellaOps.Feedser.Merge/Services/CanonicalHashCalculator.cs create mode 100644 src/StellaOps.Feedser.Merge/Services/MergeEventWriter.cs create mode 100644 src/StellaOps.Feedser.Merge/StellaOps.Feedser.Merge.csproj create mode 100644 src/StellaOps.Feedser.Merge/TASKS.md create mode 100644 src/StellaOps.Feedser.Models.Tests/AdvisoryProvenanceTests.cs create mode 100644 src/StellaOps.Feedser.Models.Tests/AdvisoryTests.cs create mode 100644 src/StellaOps.Feedser.Models.Tests/AffectedPackageStatusTests.cs create mode 100644 src/StellaOps.Feedser.Models.Tests/AliasSchemeRegistryTests.cs create mode 100644 src/StellaOps.Feedser.Models.Tests/CanonicalExampleFactory.cs create mode 100644 src/StellaOps.Feedser.Models.Tests/CanonicalExamplesTests.cs create mode 100644 src/StellaOps.Feedser.Models.Tests/CanonicalJsonSerializerTests.cs create mode 100644 src/StellaOps.Feedser.Models.Tests/Fixtures/ghsa-semver.json create mode 100644 src/StellaOps.Feedser.Models.Tests/Fixtures/kev-flag.json create mode 100644 src/StellaOps.Feedser.Models.Tests/Fixtures/nvd-basic.json create mode 100644 src/StellaOps.Feedser.Models.Tests/Fixtures/psirt-overlay.json create mode 100644 src/StellaOps.Feedser.Models.Tests/NormalizedVersionRuleTests.cs create mode 100644 src/StellaOps.Feedser.Models.Tests/OsvGhsaParityDiagnosticsTests.cs create mode 100644 src/StellaOps.Feedser.Models.Tests/OsvGhsaParityInspectorTests.cs create mode 100644 src/StellaOps.Feedser.Models.Tests/ProvenanceDiagnosticsTests.cs create mode 100644 src/StellaOps.Feedser.Models.Tests/RangePrimitivesTests.cs create mode 100644 src/StellaOps.Feedser.Models.Tests/SemVerPrimitiveTests.cs create mode 100644 src/StellaOps.Feedser.Models.Tests/SerializationDeterminismTests.cs create mode 100644 src/StellaOps.Feedser.Models.Tests/SeverityNormalizationTests.cs create mode 100644 src/StellaOps.Feedser.Models.Tests/StellaOps.Feedser.Models.Tests.csproj create mode 100644 src/StellaOps.Feedser.Models/AGENTS.md create mode 100644 src/StellaOps.Feedser.Models/Advisory.cs create mode 100644 src/StellaOps.Feedser.Models/AdvisoryCredit.cs create mode 100644 src/StellaOps.Feedser.Models/AdvisoryProvenance.cs create mode 100644 src/StellaOps.Feedser.Models/AdvisoryReference.cs create mode 100644 src/StellaOps.Feedser.Models/AffectedPackage.cs create mode 100644 src/StellaOps.Feedser.Models/AffectedPackageStatus.cs create mode 100644 src/StellaOps.Feedser.Models/AffectedPackageStatusCatalog.cs create mode 100644 src/StellaOps.Feedser.Models/AffectedVersionRange.cs create mode 100644 src/StellaOps.Feedser.Models/AffectedVersionRangeExtensions.cs create mode 100644 src/StellaOps.Feedser.Models/AliasSchemeRegistry.cs create mode 100644 src/StellaOps.Feedser.Models/AliasSchemes.cs create mode 100644 src/StellaOps.Feedser.Models/BACKWARD_COMPATIBILITY.md create mode 100644 src/StellaOps.Feedser.Models/CANONICAL_RECORDS.md create mode 100644 src/StellaOps.Feedser.Models/CanonicalJsonSerializer.cs create mode 100644 src/StellaOps.Feedser.Models/CvssMetric.cs create mode 100644 src/StellaOps.Feedser.Models/NormalizedVersionRule.cs create mode 100644 src/StellaOps.Feedser.Models/OsvGhsaParityDiagnostics.cs create mode 100644 src/StellaOps.Feedser.Models/OsvGhsaParityInspector.cs create mode 100644 src/StellaOps.Feedser.Models/PROVENANCE_GUIDELINES.md create mode 100644 src/StellaOps.Feedser.Models/ProvenanceFieldMasks.cs create mode 100644 src/StellaOps.Feedser.Models/ProvenanceInspector.cs create mode 100644 src/StellaOps.Feedser.Models/RangePrimitives.cs create mode 100644 src/StellaOps.Feedser.Models/SemVerPrimitiveExtensions.cs create mode 100644 src/StellaOps.Feedser.Models/SeverityNormalization.cs create mode 100644 src/StellaOps.Feedser.Models/SnapshotSerializer.cs create mode 100644 src/StellaOps.Feedser.Models/StellaOps.Feedser.Models.csproj create mode 100644 src/StellaOps.Feedser.Models/TASKS.md create mode 100644 src/StellaOps.Feedser.Models/Validation.cs create mode 100644 src/StellaOps.Feedser.Normalization.Tests/CpeNormalizerTests.cs create mode 100644 src/StellaOps.Feedser.Normalization.Tests/CvssMetricNormalizerTests.cs create mode 100644 src/StellaOps.Feedser.Normalization.Tests/DebianEvrParserTests.cs create mode 100644 src/StellaOps.Feedser.Normalization.Tests/DescriptionNormalizerTests.cs create mode 100644 src/StellaOps.Feedser.Normalization.Tests/NevraParserTests.cs create mode 100644 src/StellaOps.Feedser.Normalization.Tests/PackageUrlNormalizerTests.cs create mode 100644 src/StellaOps.Feedser.Normalization.Tests/SemVerRangeRuleBuilderTests.cs create mode 100644 src/StellaOps.Feedser.Normalization.Tests/StellaOps.Feedser.Normalization.Tests.csproj create mode 100644 src/StellaOps.Feedser.Normalization/AssemblyInfo.cs create mode 100644 src/StellaOps.Feedser.Normalization/Cvss/CvssMetricNormalizer.cs create mode 100644 src/StellaOps.Feedser.Normalization/Distro/DebianEvr.cs create mode 100644 src/StellaOps.Feedser.Normalization/Distro/Nevra.cs create mode 100644 src/StellaOps.Feedser.Normalization/Identifiers/Cpe23.cs create mode 100644 src/StellaOps.Feedser.Normalization/Identifiers/IdentifierNormalizer.cs create mode 100644 src/StellaOps.Feedser.Normalization/Identifiers/PackageUrl.cs create mode 100644 src/StellaOps.Feedser.Normalization/SemVer/SemVerRangeRuleBuilder.cs create mode 100644 src/StellaOps.Feedser.Normalization/StellaOps.Feedser.Normalization.csproj create mode 100644 src/StellaOps.Feedser.Normalization/TASKS.md create mode 100644 src/StellaOps.Feedser.Normalization/Text/DescriptionNormalizer.cs create mode 100644 src/StellaOps.Feedser.Source.Acsc/AGENTS.md create mode 100644 src/StellaOps.Feedser.Source.Acsc/Class1.cs create mode 100644 src/StellaOps.Feedser.Source.Acsc/StellaOps.Feedser.Source.Acsc.csproj create mode 100644 src/StellaOps.Feedser.Source.Acsc/TASKS.md create mode 100644 src/StellaOps.Feedser.Source.Cccs/AGENTS.md create mode 100644 src/StellaOps.Feedser.Source.Cccs/Class1.cs create mode 100644 src/StellaOps.Feedser.Source.Cccs/StellaOps.Feedser.Source.Cccs.csproj create mode 100644 src/StellaOps.Feedser.Source.Cccs/TASKS.md create mode 100644 src/StellaOps.Feedser.Source.CertBund/AGENTS.md create mode 100644 src/StellaOps.Feedser.Source.CertBund/Class1.cs create mode 100644 src/StellaOps.Feedser.Source.CertBund/StellaOps.Feedser.Source.CertBund.csproj create mode 100644 src/StellaOps.Feedser.Source.CertBund/TASKS.md create mode 100644 src/StellaOps.Feedser.Source.CertCc.Tests/CertCc/CertCcConnectorFetchTests.cs create mode 100644 src/StellaOps.Feedser.Source.CertCc.Tests/CertCc/CertCcConnectorSnapshotTests.cs create mode 100644 src/StellaOps.Feedser.Source.CertCc.Tests/CertCc/CertCcConnectorTests.cs create mode 100644 src/StellaOps.Feedser.Source.CertCc.Tests/Fixtures/certcc-documents.snapshot.json create mode 100644 src/StellaOps.Feedser.Source.CertCc.Tests/Fixtures/certcc-state.snapshot.json create mode 100644 src/StellaOps.Feedser.Source.CertCc.Tests/Fixtures/summary-2025-09.json create mode 100644 src/StellaOps.Feedser.Source.CertCc.Tests/Fixtures/summary-2025-10.json create mode 100644 src/StellaOps.Feedser.Source.CertCc.Tests/Fixtures/summary-2025-11.json create mode 100644 src/StellaOps.Feedser.Source.CertCc.Tests/Fixtures/summary-2025.json create mode 100644 src/StellaOps.Feedser.Source.CertCc.Tests/Fixtures/vendor-statuses-294418.json create mode 100644 src/StellaOps.Feedser.Source.CertCc.Tests/Fixtures/vendors-294418.json create mode 100644 src/StellaOps.Feedser.Source.CertCc.Tests/Fixtures/vu-257161.json create mode 100644 src/StellaOps.Feedser.Source.CertCc.Tests/Fixtures/vu-294418-vendors.json create mode 100644 src/StellaOps.Feedser.Source.CertCc.Tests/Fixtures/vu-294418-vuls.json create mode 100644 src/StellaOps.Feedser.Source.CertCc.Tests/Fixtures/vu-294418.json create mode 100644 src/StellaOps.Feedser.Source.CertCc.Tests/Fixtures/vulnerabilities-294418.json create mode 100644 src/StellaOps.Feedser.Source.CertCc.Tests/Internal/CertCcMapperTests.cs create mode 100644 src/StellaOps.Feedser.Source.CertCc.Tests/Internal/CertCcSummaryPlannerTests.cs create mode 100644 src/StellaOps.Feedser.Source.CertCc.Tests/Internal/CertCcVendorStatementParserTests.cs create mode 100644 src/StellaOps.Feedser.Source.CertCc.Tests/StellaOps.Feedser.Source.CertCc.Tests.csproj create mode 100644 src/StellaOps.Feedser.Source.CertCc/AGENTS.md create mode 100644 src/StellaOps.Feedser.Source.CertCc/CertCcConnector.cs create mode 100644 src/StellaOps.Feedser.Source.CertCc/CertCcConnectorPlugin.cs create mode 100644 src/StellaOps.Feedser.Source.CertCc/CertCcDependencyInjectionRoutine.cs create mode 100644 src/StellaOps.Feedser.Source.CertCc/CertCcServiceCollectionExtensions.cs create mode 100644 src/StellaOps.Feedser.Source.CertCc/Configuration/CertCcOptions.cs create mode 100644 src/StellaOps.Feedser.Source.CertCc/FEEDCONN-CERTCC-02-009_PLAN.md create mode 100644 src/StellaOps.Feedser.Source.CertCc/Internal/CertCcCursor.cs create mode 100644 src/StellaOps.Feedser.Source.CertCc/Internal/CertCcDiagnostics.cs create mode 100644 src/StellaOps.Feedser.Source.CertCc/Internal/CertCcMapper.cs create mode 100644 src/StellaOps.Feedser.Source.CertCc/Internal/CertCcNoteDto.cs create mode 100644 src/StellaOps.Feedser.Source.CertCc/Internal/CertCcNoteParser.cs create mode 100644 src/StellaOps.Feedser.Source.CertCc/Internal/CertCcSummaryPlan.cs create mode 100644 src/StellaOps.Feedser.Source.CertCc/Internal/CertCcSummaryPlanner.cs create mode 100644 src/StellaOps.Feedser.Source.CertCc/Internal/CertCcVendorStatementParser.cs create mode 100644 src/StellaOps.Feedser.Source.CertCc/Jobs.cs create mode 100644 src/StellaOps.Feedser.Source.CertCc/Properties/AssemblyInfo.cs create mode 100644 src/StellaOps.Feedser.Source.CertCc/README.md create mode 100644 src/StellaOps.Feedser.Source.CertCc/StellaOps.Feedser.Source.CertCc.csproj create mode 100644 src/StellaOps.Feedser.Source.CertCc/TASKS.md create mode 100644 src/StellaOps.Feedser.Source.CertFr.Tests/CertFr/CertFrConnectorTests.cs create mode 100644 src/StellaOps.Feedser.Source.CertFr.Tests/CertFr/Fixtures/certfr-advisories.snapshot.json create mode 100644 src/StellaOps.Feedser.Source.CertFr.Tests/CertFr/Fixtures/certfr-detail-AV-2024-001.html create mode 100644 src/StellaOps.Feedser.Source.CertFr.Tests/CertFr/Fixtures/certfr-detail-AV-2024-002.html create mode 100644 src/StellaOps.Feedser.Source.CertFr.Tests/CertFr/Fixtures/certfr-feed.xml create mode 100644 src/StellaOps.Feedser.Source.CertFr.Tests/StellaOps.Feedser.Source.CertFr.Tests.csproj create mode 100644 src/StellaOps.Feedser.Source.CertFr/AGENTS.md create mode 100644 src/StellaOps.Feedser.Source.CertFr/CertFrConnector.cs create mode 100644 src/StellaOps.Feedser.Source.CertFr/CertFrConnectorPlugin.cs create mode 100644 src/StellaOps.Feedser.Source.CertFr/CertFrDependencyInjectionRoutine.cs create mode 100644 src/StellaOps.Feedser.Source.CertFr/CertFrServiceCollectionExtensions.cs create mode 100644 src/StellaOps.Feedser.Source.CertFr/Configuration/CertFrOptions.cs create mode 100644 src/StellaOps.Feedser.Source.CertFr/Internal/CertFrCursor.cs create mode 100644 src/StellaOps.Feedser.Source.CertFr/Internal/CertFrDocumentMetadata.cs create mode 100644 src/StellaOps.Feedser.Source.CertFr/Internal/CertFrDto.cs create mode 100644 src/StellaOps.Feedser.Source.CertFr/Internal/CertFrFeedClient.cs create mode 100644 src/StellaOps.Feedser.Source.CertFr/Internal/CertFrFeedItem.cs create mode 100644 src/StellaOps.Feedser.Source.CertFr/Internal/CertFrMapper.cs create mode 100644 src/StellaOps.Feedser.Source.CertFr/Internal/CertFrParser.cs create mode 100644 src/StellaOps.Feedser.Source.CertFr/Jobs.cs create mode 100644 src/StellaOps.Feedser.Source.CertFr/StellaOps.Feedser.Source.CertFr.csproj create mode 100644 src/StellaOps.Feedser.Source.CertFr/TASKS.md create mode 100644 src/StellaOps.Feedser.Source.CertIn.Tests/CertIn/CertInConnectorTests.cs create mode 100644 src/StellaOps.Feedser.Source.CertIn.Tests/CertIn/Fixtures/alerts-page1.json create mode 100644 src/StellaOps.Feedser.Source.CertIn.Tests/CertIn/Fixtures/detail-CIAD-2024-0005.html create mode 100644 src/StellaOps.Feedser.Source.CertIn.Tests/CertIn/Fixtures/expected-advisory.json create mode 100644 src/StellaOps.Feedser.Source.CertIn.Tests/StellaOps.Feedser.Source.CertIn.Tests.csproj create mode 100644 src/StellaOps.Feedser.Source.CertIn/AGENTS.md create mode 100644 src/StellaOps.Feedser.Source.CertIn/CertInConnector.cs create mode 100644 src/StellaOps.Feedser.Source.CertIn/CertInConnectorPlugin.cs create mode 100644 src/StellaOps.Feedser.Source.CertIn/CertInDependencyInjectionRoutine.cs create mode 100644 src/StellaOps.Feedser.Source.CertIn/CertInServiceCollectionExtensions.cs create mode 100644 src/StellaOps.Feedser.Source.CertIn/Configuration/CertInOptions.cs create mode 100644 src/StellaOps.Feedser.Source.CertIn/Internal/CertInAdvisoryDto.cs create mode 100644 src/StellaOps.Feedser.Source.CertIn/Internal/CertInClient.cs create mode 100644 src/StellaOps.Feedser.Source.CertIn/Internal/CertInCursor.cs create mode 100644 src/StellaOps.Feedser.Source.CertIn/Internal/CertInDetailParser.cs create mode 100644 src/StellaOps.Feedser.Source.CertIn/Internal/CertInListingItem.cs create mode 100644 src/StellaOps.Feedser.Source.CertIn/Jobs.cs create mode 100644 src/StellaOps.Feedser.Source.CertIn/StellaOps.Feedser.Source.CertIn.csproj create mode 100644 src/StellaOps.Feedser.Source.CertIn/TASKS.md create mode 100644 src/StellaOps.Feedser.Source.Common.Tests/Common/CannedHttpMessageHandlerTests.cs create mode 100644 src/StellaOps.Feedser.Source.Common.Tests/Common/HtmlContentSanitizerTests.cs create mode 100644 src/StellaOps.Feedser.Source.Common.Tests/Common/PackageCoordinateHelperTests.cs create mode 100644 src/StellaOps.Feedser.Source.Common.Tests/Common/PdfTextExtractorTests.cs create mode 100644 src/StellaOps.Feedser.Source.Common.Tests/Common/SourceFetchServiceTests.cs create mode 100644 src/StellaOps.Feedser.Source.Common.Tests/Common/TimeWindowCursorPlannerTests.cs create mode 100644 src/StellaOps.Feedser.Source.Common.Tests/Common/UrlNormalizerTests.cs create mode 100644 src/StellaOps.Feedser.Source.Common.Tests/Json/JsonSchemaValidatorTests.cs create mode 100644 src/StellaOps.Feedser.Source.Common.Tests/StellaOps.Feedser.Source.Common.Tests.csproj create mode 100644 src/StellaOps.Feedser.Source.Common.Tests/Xml/XmlSchemaValidatorTests.cs create mode 100644 src/StellaOps.Feedser.Source.Common/AGENTS.md create mode 100644 src/StellaOps.Feedser.Source.Common/Cursors/PaginationPlanner.cs create mode 100644 src/StellaOps.Feedser.Source.Common/Cursors/TimeWindowCursorOptions.cs create mode 100644 src/StellaOps.Feedser.Source.Common/Cursors/TimeWindowCursorPlanner.cs create mode 100644 src/StellaOps.Feedser.Source.Common/Cursors/TimeWindowCursorState.cs create mode 100644 src/StellaOps.Feedser.Source.Common/DocumentStatuses.cs create mode 100644 src/StellaOps.Feedser.Source.Common/Fetch/CryptoJitterSource.cs create mode 100644 src/StellaOps.Feedser.Source.Common/Fetch/IJitterSource.cs create mode 100644 src/StellaOps.Feedser.Source.Common/Fetch/RawDocumentStorage.cs create mode 100644 src/StellaOps.Feedser.Source.Common/Fetch/SourceFetchContentResult.cs create mode 100644 src/StellaOps.Feedser.Source.Common/Fetch/SourceFetchRequest.cs create mode 100644 src/StellaOps.Feedser.Source.Common/Fetch/SourceFetchResult.cs create mode 100644 src/StellaOps.Feedser.Source.Common/Fetch/SourceFetchService.cs create mode 100644 src/StellaOps.Feedser.Source.Common/Fetch/SourceRetryPolicy.cs create mode 100644 src/StellaOps.Feedser.Source.Common/Html/HtmlContentSanitizer.cs create mode 100644 src/StellaOps.Feedser.Source.Common/Http/AllowlistedHttpMessageHandler.cs create mode 100644 src/StellaOps.Feedser.Source.Common/Http/ServiceCollectionExtensions.cs create mode 100644 src/StellaOps.Feedser.Source.Common/Http/SourceHttpClientOptions.cs create mode 100644 src/StellaOps.Feedser.Source.Common/Json/IJsonSchemaValidator.cs create mode 100644 src/StellaOps.Feedser.Source.Common/Json/JsonSchemaValidationError.cs create mode 100644 src/StellaOps.Feedser.Source.Common/Json/JsonSchemaValidationException.cs create mode 100644 src/StellaOps.Feedser.Source.Common/Json/JsonSchemaValidator.cs create mode 100644 src/StellaOps.Feedser.Source.Common/Packages/PackageCoordinateHelper.cs create mode 100644 src/StellaOps.Feedser.Source.Common/Pdf/PdfTextExtractor.cs create mode 100644 src/StellaOps.Feedser.Source.Common/Properties/AssemblyInfo.cs create mode 100644 src/StellaOps.Feedser.Source.Common/StellaOps.Feedser.Source.Common.csproj create mode 100644 src/StellaOps.Feedser.Source.Common/TASKS.md create mode 100644 src/StellaOps.Feedser.Source.Common/Telemetry/SourceDiagnostics.cs create mode 100644 src/StellaOps.Feedser.Source.Common/Testing/CannedHttpMessageHandler.cs create mode 100644 src/StellaOps.Feedser.Source.Common/Url/UrlNormalizer.cs create mode 100644 src/StellaOps.Feedser.Source.Common/Xml/IXmlSchemaValidator.cs create mode 100644 src/StellaOps.Feedser.Source.Common/Xml/XmlSchemaValidationError.cs create mode 100644 src/StellaOps.Feedser.Source.Common/Xml/XmlSchemaValidationException.cs create mode 100644 src/StellaOps.Feedser.Source.Common/Xml/XmlSchemaValidator.cs create mode 100644 src/StellaOps.Feedser.Source.Cve.Tests/Cve/CveConnectorTests.cs create mode 100644 src/StellaOps.Feedser.Source.Cve.Tests/Fixtures/cve-CVE-2024-0001.json create mode 100644 src/StellaOps.Feedser.Source.Cve.Tests/Fixtures/cve-list.json create mode 100644 src/StellaOps.Feedser.Source.Cve.Tests/Fixtures/expected-CVE-2024-0001.json create mode 100644 src/StellaOps.Feedser.Source.Cve.Tests/StellaOps.Feedser.Source.Cve.Tests.csproj create mode 100644 src/StellaOps.Feedser.Source.Cve/AGENTS.md create mode 100644 src/StellaOps.Feedser.Source.Cve/Configuration/CveOptions.cs create mode 100644 src/StellaOps.Feedser.Source.Cve/CveConnector.cs create mode 100644 src/StellaOps.Feedser.Source.Cve/CveConnectorPlugin.cs create mode 100644 src/StellaOps.Feedser.Source.Cve/CveDependencyInjectionRoutine.cs create mode 100644 src/StellaOps.Feedser.Source.Cve/CveServiceCollectionExtensions.cs create mode 100644 src/StellaOps.Feedser.Source.Cve/Internal/CveCursor.cs create mode 100644 src/StellaOps.Feedser.Source.Cve/Internal/CveDiagnostics.cs create mode 100644 src/StellaOps.Feedser.Source.Cve/Internal/CveListParser.cs create mode 100644 src/StellaOps.Feedser.Source.Cve/Internal/CveMapper.cs create mode 100644 src/StellaOps.Feedser.Source.Cve/Internal/CveRecordDto.cs create mode 100644 src/StellaOps.Feedser.Source.Cve/Internal/CveRecordParser.cs create mode 100644 src/StellaOps.Feedser.Source.Cve/Jobs.cs create mode 100644 src/StellaOps.Feedser.Source.Cve/StellaOps.Feedser.Source.Cve.csproj create mode 100644 src/StellaOps.Feedser.Source.Cve/TASKS.md create mode 100644 src/StellaOps.Feedser.Source.Distro.Debian.Tests/DebianConnectorTests.cs create mode 100644 src/StellaOps.Feedser.Source.Distro.Debian.Tests/DebianMapperTests.cs create mode 100644 src/StellaOps.Feedser.Source.Distro.Debian.Tests/Source/Distro/Debian/Fixtures/debian-detail-dsa-2024-123.html create mode 100644 src/StellaOps.Feedser.Source.Distro.Debian.Tests/Source/Distro/Debian/Fixtures/debian-detail-dsa-2024-124.html create mode 100644 src/StellaOps.Feedser.Source.Distro.Debian.Tests/Source/Distro/Debian/Fixtures/debian-list.txt create mode 100644 src/StellaOps.Feedser.Source.Distro.Debian.Tests/StellaOps.Feedser.Source.Distro.Debian.Tests.csproj create mode 100644 src/StellaOps.Feedser.Source.Distro.Debian/AssemblyInfo.cs create mode 100644 src/StellaOps.Feedser.Source.Distro.Debian/Configuration/DebianOptions.cs create mode 100644 src/StellaOps.Feedser.Source.Distro.Debian/DebianConnector.cs create mode 100644 src/StellaOps.Feedser.Source.Distro.Debian/DebianConnectorPlugin.cs create mode 100644 src/StellaOps.Feedser.Source.Distro.Debian/DebianDependencyInjectionRoutine.cs create mode 100644 src/StellaOps.Feedser.Source.Distro.Debian/DebianServiceCollectionExtensions.cs create mode 100644 src/StellaOps.Feedser.Source.Distro.Debian/Internal/DebianAdvisoryDto.cs create mode 100644 src/StellaOps.Feedser.Source.Distro.Debian/Internal/DebianCursor.cs create mode 100644 src/StellaOps.Feedser.Source.Distro.Debian/Internal/DebianDetailMetadata.cs create mode 100644 src/StellaOps.Feedser.Source.Distro.Debian/Internal/DebianFetchCacheEntry.cs create mode 100644 src/StellaOps.Feedser.Source.Distro.Debian/Internal/DebianHtmlParser.cs create mode 100644 src/StellaOps.Feedser.Source.Distro.Debian/Internal/DebianListEntry.cs create mode 100644 src/StellaOps.Feedser.Source.Distro.Debian/Internal/DebianListParser.cs create mode 100644 src/StellaOps.Feedser.Source.Distro.Debian/Internal/DebianMapper.cs create mode 100644 src/StellaOps.Feedser.Source.Distro.Debian/Jobs.cs create mode 100644 src/StellaOps.Feedser.Source.Distro.Debian/StellaOps.Feedser.Source.Distro.Debian.csproj create mode 100644 src/StellaOps.Feedser.Source.Distro.RedHat.Tests/RedHat/Fixtures/csaf-rhsa-2025-0001.json create mode 100644 src/StellaOps.Feedser.Source.Distro.RedHat.Tests/RedHat/Fixtures/csaf-rhsa-2025-0002.json create mode 100644 src/StellaOps.Feedser.Source.Distro.RedHat.Tests/RedHat/Fixtures/csaf-rhsa-2025-0003.json create mode 100644 src/StellaOps.Feedser.Source.Distro.RedHat.Tests/RedHat/Fixtures/rhsa-2025-0001.snapshot.json create mode 100644 src/StellaOps.Feedser.Source.Distro.RedHat.Tests/RedHat/Fixtures/rhsa-2025-0002.snapshot.json create mode 100644 src/StellaOps.Feedser.Source.Distro.RedHat.Tests/RedHat/Fixtures/rhsa-2025-0003.snapshot.json create mode 100644 src/StellaOps.Feedser.Source.Distro.RedHat.Tests/RedHat/Fixtures/summary-page1-repeat.json create mode 100644 src/StellaOps.Feedser.Source.Distro.RedHat.Tests/RedHat/Fixtures/summary-page1.json create mode 100644 src/StellaOps.Feedser.Source.Distro.RedHat.Tests/RedHat/Fixtures/summary-page2.json create mode 100644 src/StellaOps.Feedser.Source.Distro.RedHat.Tests/RedHat/Fixtures/summary-page3.json create mode 100644 src/StellaOps.Feedser.Source.Distro.RedHat.Tests/RedHat/RedHatConnectorHarnessTests.cs create mode 100644 src/StellaOps.Feedser.Source.Distro.RedHat.Tests/RedHat/RedHatConnectorTests.cs create mode 100644 src/StellaOps.Feedser.Source.Distro.RedHat.Tests/StellaOps.Feedser.Source.Distro.RedHat.Tests.csproj create mode 100644 src/StellaOps.Feedser.Source.Distro.RedHat/AGENTS.md create mode 100644 src/StellaOps.Feedser.Source.Distro.RedHat/CONFLICT_RESOLVER_NOTES.md create mode 100644 src/StellaOps.Feedser.Source.Distro.RedHat/Configuration/RedHatOptions.cs create mode 100644 src/StellaOps.Feedser.Source.Distro.RedHat/Internal/Models/RedHatCsafModels.cs create mode 100644 src/StellaOps.Feedser.Source.Distro.RedHat/Internal/RedHatCursor.cs create mode 100644 src/StellaOps.Feedser.Source.Distro.RedHat/Internal/RedHatMapper.cs create mode 100644 src/StellaOps.Feedser.Source.Distro.RedHat/Internal/RedHatSummaryItem.cs create mode 100644 src/StellaOps.Feedser.Source.Distro.RedHat/Jobs.cs create mode 100644 src/StellaOps.Feedser.Source.Distro.RedHat/Properties/AssemblyInfo.cs create mode 100644 src/StellaOps.Feedser.Source.Distro.RedHat/RedHatConnector.cs create mode 100644 src/StellaOps.Feedser.Source.Distro.RedHat/RedHatConnectorPlugin.cs create mode 100644 src/StellaOps.Feedser.Source.Distro.RedHat/RedHatDependencyInjectionRoutine.cs create mode 100644 src/StellaOps.Feedser.Source.Distro.RedHat/RedHatServiceCollectionExtensions.cs create mode 100644 src/StellaOps.Feedser.Source.Distro.RedHat/StellaOps.Feedser.Source.Distro.RedHat.csproj create mode 100644 src/StellaOps.Feedser.Source.Distro.RedHat/TASKS.md create mode 100644 src/StellaOps.Feedser.Source.Distro.Suse.Tests/Source/Distro/Suse/Fixtures/suse-changes.csv create mode 100644 src/StellaOps.Feedser.Source.Distro.Suse.Tests/Source/Distro/Suse/Fixtures/suse-su-2025_0001-1.json create mode 100644 src/StellaOps.Feedser.Source.Distro.Suse.Tests/Source/Distro/Suse/Fixtures/suse-su-2025_0002-1.json create mode 100644 src/StellaOps.Feedser.Source.Distro.Suse.Tests/StellaOps.Feedser.Source.Distro.Suse.Tests.csproj create mode 100644 src/StellaOps.Feedser.Source.Distro.Suse.Tests/SuseConnectorTests.cs create mode 100644 src/StellaOps.Feedser.Source.Distro.Suse.Tests/SuseCsafParserTests.cs create mode 100644 src/StellaOps.Feedser.Source.Distro.Suse.Tests/SuseMapperTests.cs create mode 100644 src/StellaOps.Feedser.Source.Distro.Suse/AssemblyInfo.cs create mode 100644 src/StellaOps.Feedser.Source.Distro.Suse/Configuration/SuseOptions.cs create mode 100644 src/StellaOps.Feedser.Source.Distro.Suse/Internal/SuseAdvisoryDto.cs create mode 100644 src/StellaOps.Feedser.Source.Distro.Suse/Internal/SuseChangeRecord.cs create mode 100644 src/StellaOps.Feedser.Source.Distro.Suse/Internal/SuseChangesParser.cs create mode 100644 src/StellaOps.Feedser.Source.Distro.Suse/Internal/SuseCsafParser.cs create mode 100644 src/StellaOps.Feedser.Source.Distro.Suse/Internal/SuseCursor.cs create mode 100644 src/StellaOps.Feedser.Source.Distro.Suse/Internal/SuseFetchCacheEntry.cs create mode 100644 src/StellaOps.Feedser.Source.Distro.Suse/Internal/SuseMapper.cs create mode 100644 src/StellaOps.Feedser.Source.Distro.Suse/Jobs.cs create mode 100644 src/StellaOps.Feedser.Source.Distro.Suse/StellaOps.Feedser.Source.Distro.Suse.csproj create mode 100644 src/StellaOps.Feedser.Source.Distro.Suse/SuseConnector.cs create mode 100644 src/StellaOps.Feedser.Source.Distro.Suse/SuseConnectorPlugin.cs create mode 100644 src/StellaOps.Feedser.Source.Distro.Suse/SuseDependencyInjectionRoutine.cs create mode 100644 src/StellaOps.Feedser.Source.Distro.Suse/SuseServiceCollectionExtensions.cs create mode 100644 src/StellaOps.Feedser.Source.Distro.Ubuntu.Tests/Fixtures/ubuntu-notices-page0.json create mode 100644 src/StellaOps.Feedser.Source.Distro.Ubuntu.Tests/Fixtures/ubuntu-notices-page1.json create mode 100644 src/StellaOps.Feedser.Source.Distro.Ubuntu.Tests/StellaOps.Feedser.Source.Distro.Ubuntu.Tests.csproj create mode 100644 src/StellaOps.Feedser.Source.Distro.Ubuntu.Tests/UbuntuConnectorTests.cs create mode 100644 src/StellaOps.Feedser.Source.Distro.Ubuntu/Configuration/UbuntuOptions.cs create mode 100644 src/StellaOps.Feedser.Source.Distro.Ubuntu/Internal/UbuntuCursor.cs create mode 100644 src/StellaOps.Feedser.Source.Distro.Ubuntu/Internal/UbuntuFetchCacheEntry.cs create mode 100644 src/StellaOps.Feedser.Source.Distro.Ubuntu/Internal/UbuntuMapper.cs create mode 100644 src/StellaOps.Feedser.Source.Distro.Ubuntu/Internal/UbuntuNoticeDto.cs create mode 100644 src/StellaOps.Feedser.Source.Distro.Ubuntu/Internal/UbuntuNoticeParser.cs create mode 100644 src/StellaOps.Feedser.Source.Distro.Ubuntu/Jobs.cs create mode 100644 src/StellaOps.Feedser.Source.Distro.Ubuntu/StellaOps.Feedser.Source.Distro.Ubuntu.csproj create mode 100644 src/StellaOps.Feedser.Source.Distro.Ubuntu/TASKS.md create mode 100644 src/StellaOps.Feedser.Source.Distro.Ubuntu/UbuntuConnector.cs create mode 100644 src/StellaOps.Feedser.Source.Distro.Ubuntu/UbuntuConnectorPlugin.cs create mode 100644 src/StellaOps.Feedser.Source.Distro.Ubuntu/UbuntuDependencyInjectionRoutine.cs create mode 100644 src/StellaOps.Feedser.Source.Distro.Ubuntu/UbuntuServiceCollectionExtensions.cs create mode 100644 src/StellaOps.Feedser.Source.Ghsa.Tests/Fixtures/expected-GHSA-xxxx-yyyy-zzzz.json create mode 100644 src/StellaOps.Feedser.Source.Ghsa.Tests/Fixtures/ghsa-GHSA-xxxx-yyyy-zzzz.json create mode 100644 src/StellaOps.Feedser.Source.Ghsa.Tests/Fixtures/ghsa-list.json create mode 100644 src/StellaOps.Feedser.Source.Ghsa.Tests/Ghsa/GhsaConnectorTests.cs create mode 100644 src/StellaOps.Feedser.Source.Ghsa.Tests/StellaOps.Feedser.Source.Ghsa.Tests.csproj create mode 100644 src/StellaOps.Feedser.Source.Ghsa/AGENTS.md create mode 100644 src/StellaOps.Feedser.Source.Ghsa/Configuration/GhsaOptions.cs create mode 100644 src/StellaOps.Feedser.Source.Ghsa/GhsaConnector.cs create mode 100644 src/StellaOps.Feedser.Source.Ghsa/GhsaConnectorPlugin.cs create mode 100644 src/StellaOps.Feedser.Source.Ghsa/GhsaDependencyInjectionRoutine.cs create mode 100644 src/StellaOps.Feedser.Source.Ghsa/GhsaServiceCollectionExtensions.cs create mode 100644 src/StellaOps.Feedser.Source.Ghsa/Internal/GhsaCursor.cs create mode 100644 src/StellaOps.Feedser.Source.Ghsa/Internal/GhsaDiagnostics.cs create mode 100644 src/StellaOps.Feedser.Source.Ghsa/Internal/GhsaListParser.cs create mode 100644 src/StellaOps.Feedser.Source.Ghsa/Internal/GhsaMapper.cs create mode 100644 src/StellaOps.Feedser.Source.Ghsa/Internal/GhsaRecordDto.cs create mode 100644 src/StellaOps.Feedser.Source.Ghsa/Internal/GhsaRecordParser.cs create mode 100644 src/StellaOps.Feedser.Source.Ghsa/Jobs.cs create mode 100644 src/StellaOps.Feedser.Source.Ghsa/Properties/AssemblyInfo.cs create mode 100644 src/StellaOps.Feedser.Source.Ghsa/StellaOps.Feedser.Source.Ghsa.csproj create mode 100644 src/StellaOps.Feedser.Source.Ghsa/TASKS.md create mode 100644 src/StellaOps.Feedser.Source.Ics.Cisa/AGENTS.md create mode 100644 src/StellaOps.Feedser.Source.Ics.Cisa/Class1.cs create mode 100644 src/StellaOps.Feedser.Source.Ics.Cisa/StellaOps.Feedser.Source.Ics.Cisa.csproj create mode 100644 src/StellaOps.Feedser.Source.Ics.Cisa/TASKS.md create mode 100644 src/StellaOps.Feedser.Source.Ics.Kaspersky.Tests/Kaspersky/Fixtures/detail-acme-controller-2024.html create mode 100644 src/StellaOps.Feedser.Source.Ics.Kaspersky.Tests/Kaspersky/Fixtures/expected-advisory.json create mode 100644 src/StellaOps.Feedser.Source.Ics.Kaspersky.Tests/Kaspersky/Fixtures/feed-page1.xml create mode 100644 src/StellaOps.Feedser.Source.Ics.Kaspersky.Tests/Kaspersky/KasperskyConnectorTests.cs create mode 100644 src/StellaOps.Feedser.Source.Ics.Kaspersky.Tests/StellaOps.Feedser.Source.Ics.Kaspersky.Tests.csproj create mode 100644 src/StellaOps.Feedser.Source.Ics.Kaspersky/AGENTS.md create mode 100644 src/StellaOps.Feedser.Source.Ics.Kaspersky/Configuration/KasperskyOptions.cs create mode 100644 src/StellaOps.Feedser.Source.Ics.Kaspersky/Internal/KasperskyAdvisoryDto.cs create mode 100644 src/StellaOps.Feedser.Source.Ics.Kaspersky/Internal/KasperskyAdvisoryParser.cs create mode 100644 src/StellaOps.Feedser.Source.Ics.Kaspersky/Internal/KasperskyCursor.cs create mode 100644 src/StellaOps.Feedser.Source.Ics.Kaspersky/Internal/KasperskyFeedClient.cs create mode 100644 src/StellaOps.Feedser.Source.Ics.Kaspersky/Internal/KasperskyFeedItem.cs create mode 100644 src/StellaOps.Feedser.Source.Ics.Kaspersky/Jobs.cs create mode 100644 src/StellaOps.Feedser.Source.Ics.Kaspersky/KasperskyConnector.cs create mode 100644 src/StellaOps.Feedser.Source.Ics.Kaspersky/KasperskyConnectorPlugin.cs create mode 100644 src/StellaOps.Feedser.Source.Ics.Kaspersky/KasperskyDependencyInjectionRoutine.cs create mode 100644 src/StellaOps.Feedser.Source.Ics.Kaspersky/KasperskyServiceCollectionExtensions.cs create mode 100644 src/StellaOps.Feedser.Source.Ics.Kaspersky/StellaOps.Feedser.Source.Ics.Kaspersky.csproj create mode 100644 src/StellaOps.Feedser.Source.Ics.Kaspersky/TASKS.md create mode 100644 src/StellaOps.Feedser.Source.Jvn.Tests/Jvn/Fixtures/expected-advisory.json create mode 100644 src/StellaOps.Feedser.Source.Jvn.Tests/Jvn/Fixtures/jvnrss-window1.xml create mode 100644 src/StellaOps.Feedser.Source.Jvn.Tests/Jvn/Fixtures/vuldef-JVNDB-2024-123456.xml create mode 100644 src/StellaOps.Feedser.Source.Jvn.Tests/Jvn/JvnConnectorTests.cs create mode 100644 src/StellaOps.Feedser.Source.Jvn.Tests/StellaOps.Feedser.Source.Jvn.Tests.csproj create mode 100644 src/StellaOps.Feedser.Source.Jvn/AGENTS.md create mode 100644 src/StellaOps.Feedser.Source.Jvn/Configuration/JvnOptions.cs create mode 100644 src/StellaOps.Feedser.Source.Jvn/Internal/JvnAdvisoryMapper.cs create mode 100644 src/StellaOps.Feedser.Source.Jvn/Internal/JvnConstants.cs create mode 100644 src/StellaOps.Feedser.Source.Jvn/Internal/JvnCursor.cs create mode 100644 src/StellaOps.Feedser.Source.Jvn/Internal/JvnDetailDto.cs create mode 100644 src/StellaOps.Feedser.Source.Jvn/Internal/JvnDetailParser.cs create mode 100644 src/StellaOps.Feedser.Source.Jvn/Internal/JvnOverviewItem.cs create mode 100644 src/StellaOps.Feedser.Source.Jvn/Internal/JvnOverviewPage.cs create mode 100644 src/StellaOps.Feedser.Source.Jvn/Internal/JvnSchemaProvider.cs create mode 100644 src/StellaOps.Feedser.Source.Jvn/Internal/JvnSchemaValidationException.cs create mode 100644 src/StellaOps.Feedser.Source.Jvn/Internal/MyJvnClient.cs create mode 100644 src/StellaOps.Feedser.Source.Jvn/Jobs.cs create mode 100644 src/StellaOps.Feedser.Source.Jvn/JvnConnector.cs create mode 100644 src/StellaOps.Feedser.Source.Jvn/JvnConnectorPlugin.cs create mode 100644 src/StellaOps.Feedser.Source.Jvn/JvnDependencyInjectionRoutine.cs create mode 100644 src/StellaOps.Feedser.Source.Jvn/JvnServiceCollectionExtensions.cs create mode 100644 src/StellaOps.Feedser.Source.Jvn/Schemas/data_marking.xsd create mode 100644 src/StellaOps.Feedser.Source.Jvn/Schemas/jvnrss_3.2.xsd create mode 100644 src/StellaOps.Feedser.Source.Jvn/Schemas/mod_sec_3.0.xsd create mode 100644 src/StellaOps.Feedser.Source.Jvn/Schemas/status_3.3.xsd create mode 100644 src/StellaOps.Feedser.Source.Jvn/Schemas/tlp_marking.xsd create mode 100644 src/StellaOps.Feedser.Source.Jvn/Schemas/vuldef_3.2.xsd create mode 100644 src/StellaOps.Feedser.Source.Jvn/Schemas/xml.xsd create mode 100644 src/StellaOps.Feedser.Source.Jvn/StellaOps.Feedser.Source.Jvn.csproj create mode 100644 src/StellaOps.Feedser.Source.Jvn/TASKS.md create mode 100644 src/StellaOps.Feedser.Source.Kev.Tests/Kev/Fixtures/kev-advisories.snapshot.json create mode 100644 src/StellaOps.Feedser.Source.Kev.Tests/Kev/Fixtures/kev-catalog.json create mode 100644 src/StellaOps.Feedser.Source.Kev.Tests/Kev/KevConnectorTests.cs create mode 100644 src/StellaOps.Feedser.Source.Kev.Tests/Kev/KevMapperTests.cs create mode 100644 src/StellaOps.Feedser.Source.Kev.Tests/StellaOps.Feedser.Source.Kev.Tests.csproj create mode 100644 src/StellaOps.Feedser.Source.Kev/AGENTS.md create mode 100644 src/StellaOps.Feedser.Source.Kev/Configuration/KevOptions.cs create mode 100644 src/StellaOps.Feedser.Source.Kev/Internal/KevCatalogDto.cs create mode 100644 src/StellaOps.Feedser.Source.Kev/Internal/KevCursor.cs create mode 100644 src/StellaOps.Feedser.Source.Kev/Internal/KevDiagnostics.cs create mode 100644 src/StellaOps.Feedser.Source.Kev/Internal/KevMapper.cs create mode 100644 src/StellaOps.Feedser.Source.Kev/Internal/KevSchemaProvider.cs create mode 100644 src/StellaOps.Feedser.Source.Kev/Jobs.cs create mode 100644 src/StellaOps.Feedser.Source.Kev/KevConnector.cs create mode 100644 src/StellaOps.Feedser.Source.Kev/KevConnectorPlugin.cs create mode 100644 src/StellaOps.Feedser.Source.Kev/KevDependencyInjectionRoutine.cs create mode 100644 src/StellaOps.Feedser.Source.Kev/KevServiceCollectionExtensions.cs create mode 100644 src/StellaOps.Feedser.Source.Kev/Schemas/kev-catalog.schema.json create mode 100644 src/StellaOps.Feedser.Source.Kev/StellaOps.Feedser.Source.Kev.csproj create mode 100644 src/StellaOps.Feedser.Source.Kev/TASKS.md create mode 100644 src/StellaOps.Feedser.Source.Kisa/AGENTS.md create mode 100644 src/StellaOps.Feedser.Source.Kisa/Class1.cs create mode 100644 src/StellaOps.Feedser.Source.Kisa/StellaOps.Feedser.Source.Kisa.csproj create mode 100644 src/StellaOps.Feedser.Source.Kisa/TASKS.md create mode 100644 src/StellaOps.Feedser.Source.Nvd.Tests/Nvd/Fixtures/nvd-invalid-schema.json create mode 100644 src/StellaOps.Feedser.Source.Nvd.Tests/Nvd/Fixtures/nvd-multipage-1.json create mode 100644 src/StellaOps.Feedser.Source.Nvd.Tests/Nvd/Fixtures/nvd-multipage-2.json create mode 100644 src/StellaOps.Feedser.Source.Nvd.Tests/Nvd/Fixtures/nvd-multipage-3.json create mode 100644 src/StellaOps.Feedser.Source.Nvd.Tests/Nvd/Fixtures/nvd-window-1.json create mode 100644 src/StellaOps.Feedser.Source.Nvd.Tests/Nvd/Fixtures/nvd-window-2.json create mode 100644 src/StellaOps.Feedser.Source.Nvd.Tests/Nvd/Fixtures/nvd-window-update.json create mode 100644 src/StellaOps.Feedser.Source.Nvd.Tests/Nvd/NvdConnectorHarnessTests.cs create mode 100644 src/StellaOps.Feedser.Source.Nvd.Tests/Nvd/NvdConnectorTests.cs create mode 100644 src/StellaOps.Feedser.Source.Nvd.Tests/StellaOps.Feedser.Source.Nvd.Tests.csproj create mode 100644 src/StellaOps.Feedser.Source.Nvd/AGENTS.md create mode 100644 src/StellaOps.Feedser.Source.Nvd/Configuration/NvdOptions.cs create mode 100644 src/StellaOps.Feedser.Source.Nvd/Internal/NvdCursor.cs create mode 100644 src/StellaOps.Feedser.Source.Nvd/Internal/NvdDiagnostics.cs create mode 100644 src/StellaOps.Feedser.Source.Nvd/Internal/NvdMapper.cs create mode 100644 src/StellaOps.Feedser.Source.Nvd/Internal/NvdSchemaProvider.cs create mode 100644 src/StellaOps.Feedser.Source.Nvd/NvdConnector.cs create mode 100644 src/StellaOps.Feedser.Source.Nvd/NvdConnectorPlugin.cs create mode 100644 src/StellaOps.Feedser.Source.Nvd/NvdServiceCollectionExtensions.cs create mode 100644 src/StellaOps.Feedser.Source.Nvd/Schemas/nvd-vulnerability.schema.json create mode 100644 src/StellaOps.Feedser.Source.Nvd/StellaOps.Feedser.Source.Nvd.csproj create mode 100644 src/StellaOps.Feedser.Source.Nvd/TASKS.md create mode 100644 src/StellaOps.Feedser.Source.Osv.Tests/Fixtures/osv-ghsa.ghsa.json create mode 100644 src/StellaOps.Feedser.Source.Osv.Tests/Fixtures/osv-ghsa.osv.json create mode 100644 src/StellaOps.Feedser.Source.Osv.Tests/Fixtures/osv-ghsa.raw-ghsa.json create mode 100644 src/StellaOps.Feedser.Source.Osv.Tests/Fixtures/osv-ghsa.raw-osv.json create mode 100644 src/StellaOps.Feedser.Source.Osv.Tests/Fixtures/osv-npm.snapshot.json create mode 100644 src/StellaOps.Feedser.Source.Osv.Tests/Fixtures/osv-pypi.snapshot.json create mode 100644 src/StellaOps.Feedser.Source.Osv.Tests/Osv/OsvGhsaParityRegressionTests.cs create mode 100644 src/StellaOps.Feedser.Source.Osv.Tests/Osv/OsvMapperTests.cs create mode 100644 src/StellaOps.Feedser.Source.Osv.Tests/Osv/OsvSnapshotTests.cs create mode 100644 src/StellaOps.Feedser.Source.Osv.Tests/StellaOps.Feedser.Source.Osv.Tests.csproj create mode 100644 src/StellaOps.Feedser.Source.Osv/AGENTS.md create mode 100644 src/StellaOps.Feedser.Source.Osv/Configuration/OsvOptions.cs create mode 100644 src/StellaOps.Feedser.Source.Osv/Internal/OsvCursor.cs create mode 100644 src/StellaOps.Feedser.Source.Osv/Internal/OsvMapper.cs create mode 100644 src/StellaOps.Feedser.Source.Osv/Internal/OsvVulnerabilityDto.cs create mode 100644 src/StellaOps.Feedser.Source.Osv/Jobs.cs create mode 100644 src/StellaOps.Feedser.Source.Osv/OsvConnector.cs create mode 100644 src/StellaOps.Feedser.Source.Osv/OsvConnectorPlugin.cs create mode 100644 src/StellaOps.Feedser.Source.Osv/OsvDependencyInjectionRoutine.cs create mode 100644 src/StellaOps.Feedser.Source.Osv/OsvServiceCollectionExtensions.cs create mode 100644 src/StellaOps.Feedser.Source.Osv/Properties/AssemblyInfo.cs create mode 100644 src/StellaOps.Feedser.Source.Osv/StellaOps.Feedser.Source.Osv.csproj create mode 100644 src/StellaOps.Feedser.Source.Osv/TASKS.md create mode 100644 src/StellaOps.Feedser.Source.Ru.Bdu/AGENTS.md create mode 100644 src/StellaOps.Feedser.Source.Ru.Bdu/Class1.cs create mode 100644 src/StellaOps.Feedser.Source.Ru.Bdu/StellaOps.Feedser.Source.Ru.Bdu.csproj create mode 100644 src/StellaOps.Feedser.Source.Ru.Bdu/TASKS.md create mode 100644 src/StellaOps.Feedser.Source.Ru.Nkcki/AGENTS.md create mode 100644 src/StellaOps.Feedser.Source.Ru.Nkcki/Class1.cs create mode 100644 src/StellaOps.Feedser.Source.Ru.Nkcki/StellaOps.Feedser.Source.Ru.Nkcki.csproj create mode 100644 src/StellaOps.Feedser.Source.Ru.Nkcki/TASKS.md create mode 100644 src/StellaOps.Feedser.Source.Vndr.Adobe.Tests/Adobe/AdobeConnectorFetchTests.cs create mode 100644 src/StellaOps.Feedser.Source.Vndr.Adobe.Tests/Adobe/Fixtures/adobe-advisories.snapshot.json create mode 100644 src/StellaOps.Feedser.Source.Vndr.Adobe.Tests/Adobe/Fixtures/adobe-detail-apsb25-85.html create mode 100644 src/StellaOps.Feedser.Source.Vndr.Adobe.Tests/Adobe/Fixtures/adobe-detail-apsb25-87.html create mode 100644 src/StellaOps.Feedser.Source.Vndr.Adobe.Tests/Adobe/Fixtures/adobe-index.html create mode 100644 src/StellaOps.Feedser.Source.Vndr.Adobe.Tests/StellaOps.Feedser.Source.Vndr.Adobe.Tests.csproj create mode 100644 src/StellaOps.Feedser.Source.Vndr.Adobe/AGENTS.md create mode 100644 src/StellaOps.Feedser.Source.Vndr.Adobe/AdobeConnector.cs create mode 100644 src/StellaOps.Feedser.Source.Vndr.Adobe/AdobeConnectorPlugin.cs create mode 100644 src/StellaOps.Feedser.Source.Vndr.Adobe/AdobeDiagnostics.cs create mode 100644 src/StellaOps.Feedser.Source.Vndr.Adobe/AdobeServiceCollectionExtensions.cs create mode 100644 src/StellaOps.Feedser.Source.Vndr.Adobe/Configuration/AdobeOptions.cs create mode 100644 src/StellaOps.Feedser.Source.Vndr.Adobe/Internal/AdobeBulletinDto.cs create mode 100644 src/StellaOps.Feedser.Source.Vndr.Adobe/Internal/AdobeCursor.cs create mode 100644 src/StellaOps.Feedser.Source.Vndr.Adobe/Internal/AdobeDetailParser.cs create mode 100644 src/StellaOps.Feedser.Source.Vndr.Adobe/Internal/AdobeDocumentMetadata.cs create mode 100644 src/StellaOps.Feedser.Source.Vndr.Adobe/Internal/AdobeIndexEntry.cs create mode 100644 src/StellaOps.Feedser.Source.Vndr.Adobe/Internal/AdobeIndexParser.cs create mode 100644 src/StellaOps.Feedser.Source.Vndr.Adobe/Internal/AdobeSchemaProvider.cs create mode 100644 src/StellaOps.Feedser.Source.Vndr.Adobe/Schemas/adobe-bulletin.schema.json create mode 100644 src/StellaOps.Feedser.Source.Vndr.Adobe/StellaOps.Feedser.Source.Vndr.Adobe.csproj create mode 100644 src/StellaOps.Feedser.Source.Vndr.Adobe/TASKS.md create mode 100644 src/StellaOps.Feedser.Source.Vndr.Apple.Tests/Apple/AppleConnectorTests.cs create mode 100644 src/StellaOps.Feedser.Source.Vndr.Apple.Tests/Apple/AppleFixtureManager.cs create mode 100644 src/StellaOps.Feedser.Source.Vndr.Apple.Tests/Apple/AppleLiveRegressionTests.cs create mode 100644 src/StellaOps.Feedser.Source.Vndr.Apple.Tests/Apple/Fixtures/106355.expected.json create mode 100644 src/StellaOps.Feedser.Source.Vndr.Apple.Tests/Apple/Fixtures/106355.html create mode 100644 src/StellaOps.Feedser.Source.Vndr.Apple.Tests/Apple/Fixtures/125326.expected.json create mode 100644 src/StellaOps.Feedser.Source.Vndr.Apple.Tests/Apple/Fixtures/125326.html create mode 100644 src/StellaOps.Feedser.Source.Vndr.Apple.Tests/Apple/Fixtures/125328.expected.json create mode 100644 src/StellaOps.Feedser.Source.Vndr.Apple.Tests/Apple/Fixtures/125328.html create mode 100644 src/StellaOps.Feedser.Source.Vndr.Apple.Tests/Apple/Fixtures/ht214108.html create mode 100644 src/StellaOps.Feedser.Source.Vndr.Apple.Tests/Apple/Fixtures/ht215500.html create mode 100644 src/StellaOps.Feedser.Source.Vndr.Apple.Tests/Apple/Fixtures/index.json create mode 100644 src/StellaOps.Feedser.Source.Vndr.Apple.Tests/StellaOps.Feedser.Source.Vndr.Apple.Tests.csproj create mode 100644 src/StellaOps.Feedser.Source.Vndr.Apple/AGENTS.md create mode 100644 src/StellaOps.Feedser.Source.Vndr.Apple/AppleConnector.cs create mode 100644 src/StellaOps.Feedser.Source.Vndr.Apple/AppleDependencyInjectionRoutine.cs create mode 100644 src/StellaOps.Feedser.Source.Vndr.Apple/AppleOptions.cs create mode 100644 src/StellaOps.Feedser.Source.Vndr.Apple/AppleServiceCollectionExtensions.cs create mode 100644 src/StellaOps.Feedser.Source.Vndr.Apple/Internal/AppleCursor.cs create mode 100644 src/StellaOps.Feedser.Source.Vndr.Apple/Internal/AppleDetailDto.cs create mode 100644 src/StellaOps.Feedser.Source.Vndr.Apple/Internal/AppleDetailParser.cs create mode 100644 src/StellaOps.Feedser.Source.Vndr.Apple/Internal/AppleDiagnostics.cs create mode 100644 src/StellaOps.Feedser.Source.Vndr.Apple/Internal/AppleIndexEntry.cs create mode 100644 src/StellaOps.Feedser.Source.Vndr.Apple/Internal/AppleMapper.cs create mode 100644 src/StellaOps.Feedser.Source.Vndr.Apple/Jobs.cs create mode 100644 src/StellaOps.Feedser.Source.Vndr.Apple/Properties/AssemblyInfo.cs create mode 100644 src/StellaOps.Feedser.Source.Vndr.Apple/README.md create mode 100644 src/StellaOps.Feedser.Source.Vndr.Apple/StellaOps.Feedser.Source.Vndr.Apple.csproj create mode 100644 src/StellaOps.Feedser.Source.Vndr.Apple/TASKS.md create mode 100644 src/StellaOps.Feedser.Source.Vndr.Apple/VndrAppleConnectorPlugin.cs create mode 100644 src/StellaOps.Feedser.Source.Vndr.Chromium.Tests/Chromium/ChromiumConnectorTests.cs create mode 100644 src/StellaOps.Feedser.Source.Vndr.Chromium.Tests/Chromium/ChromiumMapperTests.cs create mode 100644 src/StellaOps.Feedser.Source.Vndr.Chromium.Tests/Chromium/Fixtures/chromium-advisory.snapshot.json create mode 100644 src/StellaOps.Feedser.Source.Vndr.Chromium.Tests/Chromium/Fixtures/chromium-detail.html create mode 100644 src/StellaOps.Feedser.Source.Vndr.Chromium.Tests/Chromium/Fixtures/chromium-feed.xml create mode 100644 src/StellaOps.Feedser.Source.Vndr.Chromium.Tests/StellaOps.Feedser.Source.Vndr.Chromium.Tests.csproj create mode 100644 src/StellaOps.Feedser.Source.Vndr.Chromium/AGENTS.md create mode 100644 src/StellaOps.Feedser.Source.Vndr.Chromium/ChromiumConnector.cs create mode 100644 src/StellaOps.Feedser.Source.Vndr.Chromium/ChromiumConnectorPlugin.cs create mode 100644 src/StellaOps.Feedser.Source.Vndr.Chromium/ChromiumDiagnostics.cs create mode 100644 src/StellaOps.Feedser.Source.Vndr.Chromium/ChromiumServiceCollectionExtensions.cs create mode 100644 src/StellaOps.Feedser.Source.Vndr.Chromium/Configuration/ChromiumOptions.cs create mode 100644 src/StellaOps.Feedser.Source.Vndr.Chromium/Internal/ChromiumCursor.cs create mode 100644 src/StellaOps.Feedser.Source.Vndr.Chromium/Internal/ChromiumDocumentMetadata.cs create mode 100644 src/StellaOps.Feedser.Source.Vndr.Chromium/Internal/ChromiumDto.cs create mode 100644 src/StellaOps.Feedser.Source.Vndr.Chromium/Internal/ChromiumFeedEntry.cs create mode 100644 src/StellaOps.Feedser.Source.Vndr.Chromium/Internal/ChromiumFeedLoader.cs create mode 100644 src/StellaOps.Feedser.Source.Vndr.Chromium/Internal/ChromiumMapper.cs create mode 100644 src/StellaOps.Feedser.Source.Vndr.Chromium/Internal/ChromiumParser.cs create mode 100644 src/StellaOps.Feedser.Source.Vndr.Chromium/Internal/ChromiumSchemaProvider.cs create mode 100644 src/StellaOps.Feedser.Source.Vndr.Chromium/Properties/AssemblyInfo.cs create mode 100644 src/StellaOps.Feedser.Source.Vndr.Chromium/Schemas/chromium-post.schema.json create mode 100644 src/StellaOps.Feedser.Source.Vndr.Chromium/StellaOps.Feedser.Source.Vndr.Chromium.csproj create mode 100644 src/StellaOps.Feedser.Source.Vndr.Chromium/TASKS.md create mode 100644 src/StellaOps.Feedser.Source.Vndr.Cisco/AGENTS.md create mode 100644 src/StellaOps.Feedser.Source.Vndr.Cisco/Class1.cs create mode 100644 src/StellaOps.Feedser.Source.Vndr.Cisco/StellaOps.Feedser.Source.Vndr.Cisco.csproj create mode 100644 src/StellaOps.Feedser.Source.Vndr.Cisco/TASKS.md create mode 100644 src/StellaOps.Feedser.Source.Vndr.Msrc/AGENTS.md create mode 100644 src/StellaOps.Feedser.Source.Vndr.Msrc/Class1.cs create mode 100644 src/StellaOps.Feedser.Source.Vndr.Msrc/StellaOps.Feedser.Source.Vndr.Msrc.csproj create mode 100644 src/StellaOps.Feedser.Source.Vndr.Msrc/TASKS.md create mode 100644 src/StellaOps.Feedser.Source.Vndr.Oracle.Tests/Oracle/Fixtures/oracle-advisories.snapshot.json create mode 100644 src/StellaOps.Feedser.Source.Vndr.Oracle.Tests/Oracle/Fixtures/oracle-calendar-cpuapr2024-single.html create mode 100644 src/StellaOps.Feedser.Source.Vndr.Oracle.Tests/Oracle/Fixtures/oracle-calendar-cpuapr2024.html create mode 100644 src/StellaOps.Feedser.Source.Vndr.Oracle.Tests/Oracle/Fixtures/oracle-detail-cpuapr2024-01.html create mode 100644 src/StellaOps.Feedser.Source.Vndr.Oracle.Tests/Oracle/Fixtures/oracle-detail-cpuapr2024-02.html create mode 100644 src/StellaOps.Feedser.Source.Vndr.Oracle.Tests/Oracle/Fixtures/oracle-detail-invalid.html create mode 100644 src/StellaOps.Feedser.Source.Vndr.Oracle.Tests/Oracle/OracleConnectorTests.cs create mode 100644 src/StellaOps.Feedser.Source.Vndr.Oracle.Tests/StellaOps.Feedser.Source.Vndr.Oracle.Tests.csproj create mode 100644 src/StellaOps.Feedser.Source.Vndr.Oracle/AGENTS.md create mode 100644 src/StellaOps.Feedser.Source.Vndr.Oracle/Configuration/OracleOptions.cs create mode 100644 src/StellaOps.Feedser.Source.Vndr.Oracle/Internal/OracleAffectedEntry.cs create mode 100644 src/StellaOps.Feedser.Source.Vndr.Oracle/Internal/OracleCalendarFetcher.cs create mode 100644 src/StellaOps.Feedser.Source.Vndr.Oracle/Internal/OracleCursor.cs create mode 100644 src/StellaOps.Feedser.Source.Vndr.Oracle/Internal/OracleDocumentMetadata.cs create mode 100644 src/StellaOps.Feedser.Source.Vndr.Oracle/Internal/OracleDto.cs create mode 100644 src/StellaOps.Feedser.Source.Vndr.Oracle/Internal/OracleDtoValidator.cs create mode 100644 src/StellaOps.Feedser.Source.Vndr.Oracle/Internal/OracleMapper.cs create mode 100644 src/StellaOps.Feedser.Source.Vndr.Oracle/Internal/OracleParser.cs create mode 100644 src/StellaOps.Feedser.Source.Vndr.Oracle/Internal/OraclePatchDocument.cs create mode 100644 src/StellaOps.Feedser.Source.Vndr.Oracle/Jobs.cs create mode 100644 src/StellaOps.Feedser.Source.Vndr.Oracle/OracleConnector.cs create mode 100644 src/StellaOps.Feedser.Source.Vndr.Oracle/OracleDependencyInjectionRoutine.cs create mode 100644 src/StellaOps.Feedser.Source.Vndr.Oracle/OracleServiceCollectionExtensions.cs create mode 100644 src/StellaOps.Feedser.Source.Vndr.Oracle/Properties/AssemblyInfo.cs create mode 100644 src/StellaOps.Feedser.Source.Vndr.Oracle/StellaOps.Feedser.Source.Vndr.Oracle.csproj create mode 100644 src/StellaOps.Feedser.Source.Vndr.Oracle/TASKS.md create mode 100644 src/StellaOps.Feedser.Source.Vndr.Oracle/VndrOracleConnectorPlugin.cs create mode 100644 src/StellaOps.Feedser.Source.Vndr.Vmware.Tests/StellaOps.Feedser.Source.Vndr.Vmware.Tests.csproj create mode 100644 src/StellaOps.Feedser.Source.Vndr.Vmware.Tests/Vmware/Fixtures/vmware-advisories.snapshot.json create mode 100644 src/StellaOps.Feedser.Source.Vndr.Vmware.Tests/Vmware/Fixtures/vmware-detail-vmsa-2024-0001.json create mode 100644 src/StellaOps.Feedser.Source.Vndr.Vmware.Tests/Vmware/Fixtures/vmware-detail-vmsa-2024-0002.json create mode 100644 src/StellaOps.Feedser.Source.Vndr.Vmware.Tests/Vmware/Fixtures/vmware-detail-vmsa-2024-0003.json create mode 100644 src/StellaOps.Feedser.Source.Vndr.Vmware.Tests/Vmware/Fixtures/vmware-index-initial.json create mode 100644 src/StellaOps.Feedser.Source.Vndr.Vmware.Tests/Vmware/Fixtures/vmware-index-second.json create mode 100644 src/StellaOps.Feedser.Source.Vndr.Vmware.Tests/Vmware/VmwareConnectorTests.cs create mode 100644 src/StellaOps.Feedser.Source.Vndr.Vmware.Tests/Vmware/VmwareMapperTests.cs create mode 100644 src/StellaOps.Feedser.Source.Vndr.Vmware/AGENTS.md create mode 100644 src/StellaOps.Feedser.Source.Vndr.Vmware/Configuration/VmwareOptions.cs create mode 100644 src/StellaOps.Feedser.Source.Vndr.Vmware/Internal/VmwareCursor.cs create mode 100644 src/StellaOps.Feedser.Source.Vndr.Vmware/Internal/VmwareDetailDto.cs create mode 100644 src/StellaOps.Feedser.Source.Vndr.Vmware/Internal/VmwareFetchCacheEntry.cs create mode 100644 src/StellaOps.Feedser.Source.Vndr.Vmware/Internal/VmwareIndexItem.cs create mode 100644 src/StellaOps.Feedser.Source.Vndr.Vmware/Internal/VmwareMapper.cs create mode 100644 src/StellaOps.Feedser.Source.Vndr.Vmware/Jobs.cs create mode 100644 src/StellaOps.Feedser.Source.Vndr.Vmware/Properties/AssemblyInfo.cs create mode 100644 src/StellaOps.Feedser.Source.Vndr.Vmware/StellaOps.Feedser.Source.Vndr.Vmware.csproj create mode 100644 src/StellaOps.Feedser.Source.Vndr.Vmware/TASKS.md create mode 100644 src/StellaOps.Feedser.Source.Vndr.Vmware/VmwareConnector.cs create mode 100644 src/StellaOps.Feedser.Source.Vndr.Vmware/VmwareConnectorPlugin.cs create mode 100644 src/StellaOps.Feedser.Source.Vndr.Vmware/VmwareDependencyInjectionRoutine.cs create mode 100644 src/StellaOps.Feedser.Source.Vndr.Vmware/VmwareDiagnostics.cs create mode 100644 src/StellaOps.Feedser.Source.Vndr.Vmware/VmwareServiceCollectionExtensions.cs create mode 100644 src/StellaOps.Feedser.Storage.Mongo.Tests/AdvisoryStorePerformanceTests.cs create mode 100644 src/StellaOps.Feedser.Storage.Mongo.Tests/AdvisoryStoreTests.cs create mode 100644 src/StellaOps.Feedser.Storage.Mongo.Tests/AliasStoreTests.cs create mode 100644 src/StellaOps.Feedser.Storage.Mongo.Tests/DocumentStoreTests.cs create mode 100644 src/StellaOps.Feedser.Storage.Mongo.Tests/DtoStoreTests.cs create mode 100644 src/StellaOps.Feedser.Storage.Mongo.Tests/ExportStateManagerTests.cs create mode 100644 src/StellaOps.Feedser.Storage.Mongo.Tests/ExportStateStoreTests.cs create mode 100644 src/StellaOps.Feedser.Storage.Mongo.Tests/MergeEventStoreTests.cs create mode 100644 src/StellaOps.Feedser.Storage.Mongo.Tests/Migrations/MongoMigrationRunnerTests.cs create mode 100644 src/StellaOps.Feedser.Storage.Mongo.Tests/MongoJobStoreTests.cs create mode 100644 src/StellaOps.Feedser.Storage.Mongo.Tests/MongoSourceStateRepositoryTests.cs create mode 100644 src/StellaOps.Feedser.Storage.Mongo.Tests/RawDocumentRetentionServiceTests.cs create mode 100644 src/StellaOps.Feedser.Storage.Mongo.Tests/StellaOps.Feedser.Storage.Mongo.Tests.csproj create mode 100644 src/StellaOps.Feedser.Storage.Mongo/AGENTS.md create mode 100644 src/StellaOps.Feedser.Storage.Mongo/Advisories/AdvisoryDocument.cs create mode 100644 src/StellaOps.Feedser.Storage.Mongo/Advisories/AdvisoryStore.cs create mode 100644 src/StellaOps.Feedser.Storage.Mongo/Advisories/IAdvisoryStore.cs create mode 100644 src/StellaOps.Feedser.Storage.Mongo/Advisories/NormalizedVersionDocument.cs create mode 100644 src/StellaOps.Feedser.Storage.Mongo/Advisories/NormalizedVersionDocumentFactory.cs create mode 100644 src/StellaOps.Feedser.Storage.Mongo/Aliases/AliasDocument.cs create mode 100644 src/StellaOps.Feedser.Storage.Mongo/Aliases/AliasStore.cs create mode 100644 src/StellaOps.Feedser.Storage.Mongo/Aliases/AliasStoreConstants.cs create mode 100644 src/StellaOps.Feedser.Storage.Mongo/Aliases/AliasStoreMetrics.cs create mode 100644 src/StellaOps.Feedser.Storage.Mongo/Aliases/IAliasStore.cs create mode 100644 src/StellaOps.Feedser.Storage.Mongo/ChangeHistory/ChangeHistoryDocument.cs create mode 100644 src/StellaOps.Feedser.Storage.Mongo/ChangeHistory/ChangeHistoryDocumentExtensions.cs create mode 100644 src/StellaOps.Feedser.Storage.Mongo/ChangeHistory/ChangeHistoryFieldChange.cs create mode 100644 src/StellaOps.Feedser.Storage.Mongo/ChangeHistory/ChangeHistoryRecord.cs create mode 100644 src/StellaOps.Feedser.Storage.Mongo/ChangeHistory/IChangeHistoryStore.cs create mode 100644 src/StellaOps.Feedser.Storage.Mongo/ChangeHistory/MongoChangeHistoryStore.cs create mode 100644 src/StellaOps.Feedser.Storage.Mongo/Documents/DocumentDocument.cs create mode 100644 src/StellaOps.Feedser.Storage.Mongo/Documents/DocumentRecord.cs create mode 100644 src/StellaOps.Feedser.Storage.Mongo/Documents/DocumentStore.cs create mode 100644 src/StellaOps.Feedser.Storage.Mongo/Documents/IDocumentStore.cs create mode 100644 src/StellaOps.Feedser.Storage.Mongo/Dtos/DtoDocument.cs create mode 100644 src/StellaOps.Feedser.Storage.Mongo/Dtos/DtoRecord.cs create mode 100644 src/StellaOps.Feedser.Storage.Mongo/Dtos/DtoStore.cs create mode 100644 src/StellaOps.Feedser.Storage.Mongo/Dtos/IDtoStore.cs create mode 100644 src/StellaOps.Feedser.Storage.Mongo/Exporting/ExportStateDocument.cs create mode 100644 src/StellaOps.Feedser.Storage.Mongo/Exporting/ExportStateManager.cs create mode 100644 src/StellaOps.Feedser.Storage.Mongo/Exporting/ExportStateRecord.cs create mode 100644 src/StellaOps.Feedser.Storage.Mongo/Exporting/ExportStateStore.cs create mode 100644 src/StellaOps.Feedser.Storage.Mongo/Exporting/IExportStateStore.cs create mode 100644 src/StellaOps.Feedser.Storage.Mongo/ISourceStateRepository.cs create mode 100644 src/StellaOps.Feedser.Storage.Mongo/JobLeaseDocument.cs create mode 100644 src/StellaOps.Feedser.Storage.Mongo/JobRunDocument.cs create mode 100644 src/StellaOps.Feedser.Storage.Mongo/JpFlags/IJpFlagStore.cs create mode 100644 src/StellaOps.Feedser.Storage.Mongo/JpFlags/JpFlagDocument.cs create mode 100644 src/StellaOps.Feedser.Storage.Mongo/JpFlags/JpFlagRecord.cs create mode 100644 src/StellaOps.Feedser.Storage.Mongo/JpFlags/JpFlagStore.cs create mode 100644 src/StellaOps.Feedser.Storage.Mongo/MIGRATIONS.md create mode 100644 src/StellaOps.Feedser.Storage.Mongo/MergeEvents/IMergeEventStore.cs create mode 100644 src/StellaOps.Feedser.Storage.Mongo/MergeEvents/MergeEventDocument.cs create mode 100644 src/StellaOps.Feedser.Storage.Mongo/MergeEvents/MergeEventRecord.cs create mode 100644 src/StellaOps.Feedser.Storage.Mongo/MergeEvents/MergeEventStore.cs create mode 100644 src/StellaOps.Feedser.Storage.Mongo/MergeEvents/MergeFieldDecision.cs create mode 100644 src/StellaOps.Feedser.Storage.Mongo/Migrations/EnsureDocumentExpiryIndexesMigration.cs create mode 100644 src/StellaOps.Feedser.Storage.Mongo/Migrations/EnsureGridFsExpiryIndexesMigration.cs create mode 100644 src/StellaOps.Feedser.Storage.Mongo/Migrations/IMongoMigration.cs create mode 100644 src/StellaOps.Feedser.Storage.Mongo/Migrations/MongoMigrationDocument.cs create mode 100644 src/StellaOps.Feedser.Storage.Mongo/Migrations/MongoMigrationRunner.cs create mode 100644 src/StellaOps.Feedser.Storage.Mongo/Migrations/SemVerStyleBackfillMigration.cs create mode 100644 src/StellaOps.Feedser.Storage.Mongo/MongoBootstrapper.cs create mode 100644 src/StellaOps.Feedser.Storage.Mongo/MongoJobStore.cs create mode 100644 src/StellaOps.Feedser.Storage.Mongo/MongoLeaseStore.cs create mode 100644 src/StellaOps.Feedser.Storage.Mongo/MongoSourceStateRepository.cs create mode 100644 src/StellaOps.Feedser.Storage.Mongo/MongoStorageDefaults.cs create mode 100644 src/StellaOps.Feedser.Storage.Mongo/MongoStorageOptions.cs create mode 100644 src/StellaOps.Feedser.Storage.Mongo/Properties/AssemblyInfo.cs create mode 100644 src/StellaOps.Feedser.Storage.Mongo/PsirtFlags/IPsirtFlagStore.cs create mode 100644 src/StellaOps.Feedser.Storage.Mongo/PsirtFlags/PsirtFlagDocument.cs create mode 100644 src/StellaOps.Feedser.Storage.Mongo/PsirtFlags/PsirtFlagRecord.cs create mode 100644 src/StellaOps.Feedser.Storage.Mongo/PsirtFlags/PsirtFlagStore.cs create mode 100644 src/StellaOps.Feedser.Storage.Mongo/RawDocumentRetentionService.cs create mode 100644 src/StellaOps.Feedser.Storage.Mongo/ServiceCollectionExtensions.cs create mode 100644 src/StellaOps.Feedser.Storage.Mongo/SourceStateDocument.cs create mode 100644 src/StellaOps.Feedser.Storage.Mongo/SourceStateRecord.cs create mode 100644 src/StellaOps.Feedser.Storage.Mongo/SourceStateRepositoryExtensions.cs create mode 100644 src/StellaOps.Feedser.Storage.Mongo/StellaOps.Feedser.Storage.Mongo.csproj create mode 100644 src/StellaOps.Feedser.Storage.Mongo/TASKS.md create mode 100644 src/StellaOps.Feedser.Testing/ConnectorTestHarness.cs create mode 100644 src/StellaOps.Feedser.Testing/MongoIntegrationFixture.cs create mode 100644 src/StellaOps.Feedser.Testing/StellaOps.Feedser.Testing.csproj create mode 100644 src/StellaOps.Feedser.Tests.Shared/AssemblyInfo.cs create mode 100644 src/StellaOps.Feedser.Tests.Shared/MongoFixtureCollection.cs create mode 100644 src/StellaOps.Feedser.WebService.Tests/FeedserOptionsPostConfigureTests.cs create mode 100644 src/StellaOps.Feedser.WebService.Tests/PluginLoaderTests.cs create mode 100644 src/StellaOps.Feedser.WebService.Tests/StellaOps.Feedser.WebService.Tests.csproj create mode 100644 src/StellaOps.Feedser.WebService.Tests/WebServiceEndpointsTests.cs create mode 100644 src/StellaOps.Feedser.WebService/AGENTS.md create mode 100644 src/StellaOps.Feedser.WebService/Diagnostics/HealthContracts.cs create mode 100644 src/StellaOps.Feedser.WebService/Diagnostics/JobMetrics.cs create mode 100644 src/StellaOps.Feedser.WebService/Diagnostics/ProblemTypes.cs create mode 100644 src/StellaOps.Feedser.WebService/Diagnostics/ServiceStatus.cs create mode 100644 src/StellaOps.Feedser.WebService/Extensions/ConfigurationExtensions.cs create mode 100644 src/StellaOps.Feedser.WebService/Extensions/JobRegistrationExtensions.cs create mode 100644 src/StellaOps.Feedser.WebService/Extensions/TelemetryExtensions.cs create mode 100644 src/StellaOps.Feedser.WebService/Filters/JobAuthorizationAuditFilter.cs create mode 100644 src/StellaOps.Feedser.WebService/Jobs/JobDefinitionResponse.cs create mode 100644 src/StellaOps.Feedser.WebService/Jobs/JobRunResponse.cs create mode 100644 src/StellaOps.Feedser.WebService/Jobs/JobTriggerRequest.cs create mode 100644 src/StellaOps.Feedser.WebService/Options/FeedserOptions.cs create mode 100644 src/StellaOps.Feedser.WebService/Options/FeedserOptionsPostConfigure.cs create mode 100644 src/StellaOps.Feedser.WebService/Options/FeedserOptionsValidator.cs create mode 100644 src/StellaOps.Feedser.WebService/Program.cs create mode 100644 src/StellaOps.Feedser.WebService/Properties/launchSettings.json create mode 100644 src/StellaOps.Feedser.WebService/StellaOps.Feedser.WebService.csproj create mode 100644 src/StellaOps.Feedser.WebService/TASKS.md create mode 100644 src/StellaOps.Feedser.sln create mode 100644 src/StellaOps.Plugin/DependencyInjection/PluginDependencyInjectionExtensions.cs create mode 100644 src/StellaOps.Plugin/DependencyInjection/StellaOpsPluginRegistration.cs create mode 100644 src/StellaOps.Plugin/Hosting/PluginAssembly.cs create mode 100644 src/StellaOps.Plugin/Hosting/PluginHost.cs create mode 100644 src/StellaOps.Plugin/Hosting/PluginHostOptions.cs create mode 100644 src/StellaOps.Plugin/Hosting/PluginHostResult.cs create mode 100644 src/StellaOps.Plugin/Hosting/PluginLoadContext.cs create mode 100644 src/StellaOps.Plugin/Internal/ReflectionExtensions.cs create mode 100644 src/StellaOps.Plugin/PluginContracts.cs create mode 100644 src/StellaOps.Plugin/StellaOps.Plugin.csproj create mode 100644 src/StellaOps.Web/AGENTS.md create mode 100644 src/StellaOps.Web/TASKS.md create mode 100644 src/StellaOps.sln create mode 100644 src/farewell.txt create mode 100644 src/global.json create mode 100644 tools/FixtureUpdater/FixtureUpdater.csproj create mode 100644 tools/FixtureUpdater/Program.cs diff --git a/.gitea/workflows/_deprecated-feedser-ci.yml.disabled b/.gitea/workflows/_deprecated-feedser-ci.yml.disabled new file mode 100644 index 00000000..781a3c85 --- /dev/null +++ b/.gitea/workflows/_deprecated-feedser-ci.yml.disabled @@ -0,0 +1,29 @@ +name: Feedser CI + +on: + push: + branches: ["main", "develop"] + pull_request: + branches: ["main", "develop"] + +jobs: + build-and-test: + runs-on: ubuntu-latest + steps: + - name: Check out repository + uses: actions/checkout@v4 + + - name: Setup .NET 10 preview + uses: actions/setup-dotnet@v4 + with: + dotnet-version: 10.0.100-rc.1.25451.107 + include-prerelease: true + + - name: Restore dependencies + run: dotnet restore src/StellaOps.Feedser/StellaOps.Feedser.sln + + - name: Build + run: dotnet build src/StellaOps.Feedser/StellaOps.Feedser.sln --configuration Release --no-restore -warnaserror + + - name: Test + run: dotnet test src/StellaOps.Feedser/StellaOps.Feedser.Tests/StellaOps.Feedser.Tests.csproj --configuration Release --no-restore --logger "trx;LogFileName=feedser-tests.trx" diff --git a/.gitea/workflows/_deprecated-feedser-tests.yml.disabled b/.gitea/workflows/_deprecated-feedser-tests.yml.disabled new file mode 100644 index 00000000..c831a15d --- /dev/null +++ b/.gitea/workflows/_deprecated-feedser-tests.yml.disabled @@ -0,0 +1,87 @@ +name: Feedser Tests CI + +on: + push: + paths: + - 'StellaOps.Feedser/**' + - '.gitea/workflows/feedser-tests.yml' + pull_request: + paths: + - 'StellaOps.Feedser/**' + - '.gitea/workflows/feedser-tests.yml' + +jobs: + advisory-store-performance: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + + - name: Set up .NET SDK + uses: actions/setup-dotnet@v4 + with: + dotnet-version: 10.0.100-rc.1 + + - name: Restore dependencies + working-directory: StellaOps.Feedser + run: dotnet restore StellaOps.Feedser.Tests/StellaOps.Feedser.Tests.csproj + + - name: Run advisory store performance test + working-directory: StellaOps.Feedser + run: | + set -euo pipefail + dotnet test \ + StellaOps.Feedser.Tests/StellaOps.Feedser.Tests.csproj \ + --filter "FullyQualifiedName~AdvisoryStorePerformanceTests" \ + --logger:"console;verbosity=detailed" | tee performance.log + + - name: Upload performance log + if: always() + uses: actions/upload-artifact@v4 + with: + name: advisory-store-performance-log + path: StellaOps.Feedser/performance.log + + full-test-suite: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + + - name: Set up .NET SDK + uses: actions/setup-dotnet@v4 + with: + dotnet-version: 10.0.100-rc.1 + + - name: Restore dependencies + working-directory: StellaOps.Feedser + run: dotnet restore StellaOps.Feedser.Tests/StellaOps.Feedser.Tests.csproj + + - name: Run full test suite with baseline guard + working-directory: StellaOps.Feedser + env: + BASELINE_SECONDS: "19.8" + TOLERANCE_PERCENT: "25" + run: | + set -euo pipefail + start=$(date +%s) + dotnet test StellaOps.Feedser.Tests/StellaOps.Feedser.Tests.csproj --no-build | tee full-tests.log + end=$(date +%s) + duration=$((end-start)) + echo "Full test duration: ${duration}s" + export DURATION_SECONDS="$duration" + python - <<'PY' +import os, sys +duration = float(os.environ["DURATION_SECONDS"]) +baseline = float(os.environ["BASELINE_SECONDS"]) +tolerance = float(os.environ["TOLERANCE_PERCENT"]) +threshold = baseline * (1 + tolerance / 100) +print(f"Baseline {baseline:.1f}s, threshold {threshold:.1f}s, observed {duration:.1f}s") +if duration > threshold: + sys.exit(f"Full test duration {duration:.1f}s exceeded threshold {threshold:.1f}s") +PY + + - name: Upload full test log + if: always() + uses: actions/upload-artifact@v4 + with: + name: full-test-suite-log + path: StellaOps.Feedser/full-tests.log diff --git a/.gitea/workflows/build-test-deploy.yml b/.gitea/workflows/build-test-deploy.yml new file mode 100644 index 00000000..09b4287a --- /dev/null +++ b/.gitea/workflows/build-test-deploy.yml @@ -0,0 +1,341 @@ +# .gitea/workflows/build-test-deploy.yml +# Unified CI/CD workflow for git.stella-ops.org (Feedser monorepo) + +name: Build Test Deploy + +on: + push: + branches: [ main ] + paths: + - 'src/**' + - 'docs/**' + - 'scripts/**' + - 'Directory.Build.props' + - 'Directory.Build.targets' + - 'global.json' + - '.gitea/workflows/**' + pull_request: + branches: [ main, develop ] + paths: + - 'src/**' + - 'docs/**' + - 'scripts/**' + - '.gitea/workflows/**' + workflow_dispatch: + inputs: + force_deploy: + description: 'Ignore branch checks and run the deploy stage' + required: false + default: 'false' + type: boolean + +env: + DOTNET_VERSION: '10.0.100-rc.1.25451.107' + BUILD_CONFIGURATION: Release + CI_CACHE_ROOT: /data/.cache/stella-ops/feedser + RUNNER_TOOL_CACHE: /toolcache + +jobs: + build-test: + runs-on: ubuntu-22.04 + environment: ${{ github.event_name == 'pull_request' && 'preview' || 'staging' }} + env: + PUBLISH_DIR: ${{ github.workspace }}/artifacts/publish/webservice + AUTHORITY_PUBLISH_DIR: ${{ github.workspace }}/artifacts/publish/authority + TEST_RESULTS_DIR: ${{ github.workspace }}/artifacts/test-results + steps: + - name: Checkout repository + uses: actions/checkout@v4 + with: + fetch-depth: 0 + + - name: Setup .NET ${{ env.DOTNET_VERSION }} + uses: actions/setup-dotnet@v4 + with: + dotnet-version: ${{ env.DOTNET_VERSION }} + include-prerelease: true + + - name: Restore dependencies + run: dotnet restore src/StellaOps.Feedser.sln + + - name: Build solution (warnings as errors) + run: dotnet build src/StellaOps.Feedser.sln --configuration $BUILD_CONFIGURATION --no-restore -warnaserror + + - name: Run unit and integration tests + run: | + mkdir -p "$TEST_RESULTS_DIR" + dotnet test src/StellaOps.Feedser.sln \ + --configuration $BUILD_CONFIGURATION \ + --no-build \ + --logger "trx;LogFileName=stellaops-feedser-tests.trx" \ + --results-directory "$TEST_RESULTS_DIR" + + - name: Publish Feedser web service + run: | + mkdir -p "$PUBLISH_DIR" + dotnet publish src/StellaOps.Feedser.WebService/StellaOps.Feedser.WebService.csproj \ + --configuration $BUILD_CONFIGURATION \ + --no-build \ + --output "$PUBLISH_DIR" + + - name: Upload published artifacts + uses: actions/upload-artifact@v4 + with: + name: feedser-publish + path: ${{ env.PUBLISH_DIR }} + if-no-files-found: error + retention-days: 7 + + - name: Restore Authority solution + run: dotnet restore src/StellaOps.Authority/StellaOps.Authority.sln + + - name: Build Authority solution + run: dotnet build src/StellaOps.Authority/StellaOps.Authority.sln --configuration $BUILD_CONFIGURATION --no-restore -warnaserror + + - name: Run Authority tests + run: | + dotnet test src/StellaOps.Configuration.Tests/StellaOps.Configuration.Tests.csproj \ + --configuration $BUILD_CONFIGURATION \ + --no-build \ + --logger "trx;LogFileName=stellaops-authority-tests.trx" \ + --results-directory "$TEST_RESULTS_DIR" + + - name: Publish Authority web service + run: | + mkdir -p "$AUTHORITY_PUBLISH_DIR" + dotnet publish src/StellaOps.Authority/StellaOps.Authority/StellaOps.Authority.csproj \ + --configuration $BUILD_CONFIGURATION \ + --no-build \ + --output "$AUTHORITY_PUBLISH_DIR" + + - name: Upload Authority artifacts + uses: actions/upload-artifact@v4 + with: + name: authority-publish + path: ${{ env.AUTHORITY_PUBLISH_DIR }} + if-no-files-found: error + retention-days: 7 + + - name: Upload test results + if: always() + uses: actions/upload-artifact@v4 + with: + name: feedser-test-results + path: ${{ env.TEST_RESULTS_DIR }} + if-no-files-found: ignore + retention-days: 7 + + authority-container: + runs-on: ubuntu-22.04 + needs: build-test + steps: + - name: Checkout repository + uses: actions/checkout@v4 + + - name: Validate Authority compose file + run: docker compose -f ops/authority/docker-compose.authority.yaml config + + - name: Build Authority container image + run: docker build -f ops/authority/Dockerfile -t stellaops-authority:ci . + + docs: + runs-on: ubuntu-22.04 + env: + DOCS_OUTPUT_DIR: ${{ github.workspace }}/artifacts/docs-site + steps: + - name: Checkout repository + uses: actions/checkout@v4 + + - name: Setup Python + uses: actions/setup-python@v5 + with: + python-version: '3.11' + + - name: Install documentation dependencies + run: | + python -m pip install --upgrade pip + python -m pip install markdown pygments + + - name: Render documentation bundle + run: | + python scripts/render_docs.py --source docs --output "$DOCS_OUTPUT_DIR" --clean + + - name: Upload documentation artifact + uses: actions/upload-artifact@v4 + with: + name: feedser-docs-site + path: ${{ env.DOCS_OUTPUT_DIR }} + if-no-files-found: error + retention-days: 7 + + deploy: + runs-on: ubuntu-22.04 + needs: [build-test, docs] + if: >- + needs.build-test.result == 'success' && + needs.docs.result == 'success' && + ( + (github.event_name == 'push' && github.ref == 'refs/heads/main') || + github.event_name == 'workflow_dispatch' + ) + environment: staging + steps: + - name: Checkout repository + uses: actions/checkout@v4 + with: + sparse-checkout: | + scripts + .gitea/workflows + sparse-checkout-cone-mode: true + + - name: Check if deployment should proceed + id: check-deploy + run: | + if [ "${{ github.event_name }}" = "workflow_dispatch" ]; then + if [ "${{ github.event.inputs.force_deploy }}" = "true" ]; then + echo "should-deploy=true" >> $GITHUB_OUTPUT + echo "✅ Manual deployment requested" + else + echo "should-deploy=false" >> $GITHUB_OUTPUT + echo "ℹ️ Manual dispatch without force_deploy=true — skipping" + fi + elif [ "${{ github.ref }}" = "refs/heads/main" ]; then + echo "should-deploy=true" >> $GITHUB_OUTPUT + echo "✅ Deploying latest main branch build" + else + echo "should-deploy=false" >> $GITHUB_OUTPUT + echo "ℹ️ Deployment restricted to main branch" + fi + + - name: Resolve deployment credentials + id: params + if: steps.check-deploy.outputs.should-deploy == 'true' + run: | + missing=() + + host="${{ secrets.STAGING_DEPLOYMENT_HOST }}" + if [ -z "$host" ]; then host="${{ vars.STAGING_DEPLOYMENT_HOST }}"; fi + if [ -z "$host" ]; then host="${{ secrets.DEPLOYMENT_HOST }}"; fi + if [ -z "$host" ]; then host="${{ vars.DEPLOYMENT_HOST }}"; fi + if [ -z "$host" ]; then missing+=("STAGING_DEPLOYMENT_HOST"); fi + + user="${{ secrets.STAGING_DEPLOYMENT_USERNAME }}" + if [ -z "$user" ]; then user="${{ vars.STAGING_DEPLOYMENT_USERNAME }}"; fi + if [ -z "$user" ]; then user="${{ secrets.DEPLOYMENT_USERNAME }}"; fi + if [ -z "$user" ]; then user="${{ vars.DEPLOYMENT_USERNAME }}"; fi + if [ -z "$user" ]; then missing+=("STAGING_DEPLOYMENT_USERNAME"); fi + + path="${{ secrets.STAGING_DEPLOYMENT_PATH }}" + if [ -z "$path" ]; then path="${{ vars.STAGING_DEPLOYMENT_PATH }}"; fi + + docs_path="${{ secrets.STAGING_DOCS_PATH }}" + if [ -z "$docs_path" ]; then docs_path="${{ vars.STAGING_DOCS_PATH }}"; fi + + key="${{ secrets.STAGING_DEPLOYMENT_KEY }}" + if [ -z "$key" ]; then key="${{ secrets.DEPLOYMENT_KEY }}"; fi + if [ -z "$key" ]; then key="${{ vars.STAGING_DEPLOYMENT_KEY }}"; fi + if [ -z "$key" ]; then key="${{ vars.DEPLOYMENT_KEY }}"; fi + if [ -z "$key" ]; then missing+=("STAGING_DEPLOYMENT_KEY"); fi + + if [ ${#missing[@]} -gt 0 ]; then + echo "❌ Missing deployment configuration: ${missing[*]}" + exit 1 + fi + + key_file="$RUNNER_TEMP/staging_deploy_key" + printf '%s\n' "$key" > "$key_file" + chmod 600 "$key_file" + + echo "host=$host" >> $GITHUB_OUTPUT + echo "user=$user" >> $GITHUB_OUTPUT + echo "path=$path" >> $GITHUB_OUTPUT + echo "docs-path=$docs_path" >> $GITHUB_OUTPUT + echo "key-file=$key_file" >> $GITHUB_OUTPUT + + - name: Download service artifact + if: steps.check-deploy.outputs.should-deploy == 'true' && steps.params.outputs.path != '' + uses: actions/download-artifact@v4 + with: + name: feedser-publish + path: artifacts/service + + - name: Download documentation artifact + if: steps.check-deploy.outputs.should-deploy == 'true' && steps.params.outputs['docs-path'] != '' + uses: actions/download-artifact@v4 + with: + name: feedser-docs-site + path: artifacts/docs + + - name: Install rsync + if: steps.check-deploy.outputs.should-deploy == 'true' + run: | + if command -v rsync >/dev/null 2>&1; then + exit 0 + fi + CACHE_DIR="${CI_CACHE_ROOT:-/tmp}/apt" + mkdir -p "$CACHE_DIR" + KEY="rsync-$(lsb_release -rs 2>/dev/null || echo unknown)" + DEB_DIR="$CACHE_DIR/$KEY" + mkdir -p "$DEB_DIR" + if ls "$DEB_DIR"/rsync*.deb >/dev/null 2>&1; then + apt-get update + apt-get install -y --no-install-recommends "$DEB_DIR"/libpopt0*.deb "$DEB_DIR"/rsync*.deb + else + apt-get update + apt-get download rsync libpopt0 + mv rsync*.deb libpopt0*.deb "$DEB_DIR"/ + dpkg -i "$DEB_DIR"/libpopt0*.deb "$DEB_DIR"/rsync*.deb || apt-get install -f -y + fi + + - name: Deploy service bundle + if: steps.check-deploy.outputs.should-deploy == 'true' && steps.params.outputs.path != '' + env: + HOST: ${{ steps.params.outputs.host }} + USER: ${{ steps.params.outputs.user }} + TARGET: ${{ steps.params.outputs.path }} + KEY_FILE: ${{ steps.params.outputs['key-file'] }} + run: | + SERVICE_DIR="artifacts/service/feedser-publish" + if [ ! -d "$SERVICE_DIR" ]; then + echo "❌ Service artifact directory missing ($SERVICE_DIR)" + exit 1 + fi + echo "🚀 Deploying Feedser web service to $HOST:$TARGET" + rsync -az --delete \ + -e "ssh -i $KEY_FILE -o StrictHostKeyChecking=no" \ + "$SERVICE_DIR"/ \ + "$USER@$HOST:$TARGET/" + + - name: Deploy documentation bundle + if: steps.check-deploy.outputs.should-deploy == 'true' && steps.params.outputs['docs-path'] != '' + env: + HOST: ${{ steps.params.outputs.host }} + USER: ${{ steps.params.outputs.user }} + DOCS_TARGET: ${{ steps.params.outputs['docs-path'] }} + KEY_FILE: ${{ steps.params.outputs['key-file'] }} + run: | + DOCS_DIR="artifacts/docs/feedser-docs-site" + if [ ! -d "$DOCS_DIR" ]; then + echo "❌ Documentation artifact directory missing ($DOCS_DIR)" + exit 1 + fi + echo "📚 Deploying documentation bundle to $HOST:$DOCS_TARGET" + rsync -az --delete \ + -e "ssh -i $KEY_FILE -o StrictHostKeyChecking=no" \ + "$DOCS_DIR"/ \ + "$USER@$HOST:$DOCS_TARGET/" + + - name: Deployment summary + if: steps.check-deploy.outputs.should-deploy == 'true' + run: | + echo "✅ Deployment completed" + echo " Host: ${{ steps.params.outputs.host }}" + echo " Service path: ${{ steps.params.outputs.path || '(skipped)' }}" + echo " Docs path: ${{ steps.params.outputs['docs-path'] || '(skipped)' }}" + + - name: Deployment skipped summary + if: steps.check-deploy.outputs.should-deploy != 'true' + run: | + echo "ℹ️ Deployment stage skipped" + echo " Event: ${{ github.event_name }}" + echo " Ref: ${{ github.ref }}" diff --git a/.gitea/workflows/docs.yml b/.gitea/workflows/docs.yml new file mode 100755 index 00000000..c7e19ad1 --- /dev/null +++ b/.gitea/workflows/docs.yml @@ -0,0 +1,70 @@ +# .gitea/workflows/docs.yml +# Documentation quality checks and preview artefacts + +name: Docs CI + +on: + push: + paths: + - 'docs/**' + - 'scripts/render_docs.py' + - '.gitea/workflows/docs.yml' + pull_request: + paths: + - 'docs/**' + - 'scripts/render_docs.py' + - '.gitea/workflows/docs.yml' + workflow_dispatch: {} + +env: + NODE_VERSION: '20' + PYTHON_VERSION: '3.11' + +jobs: + lint-and-preview: + runs-on: ubuntu-22.04 + env: + DOCS_OUTPUT_DIR: ${{ github.workspace }}/artifacts/docs-preview + steps: + - name: Checkout repository + uses: actions/checkout@v4 + + - name: Setup Node.js + uses: actions/setup-node@v4 + with: + node-version: ${{ env.NODE_VERSION }} + + - name: Install markdown linters + run: | + npm install markdown-link-check remark-cli remark-preset-lint-recommended + + - name: Link check + run: | + find docs -name '*.md' -print0 | \ + xargs -0 -n1 -I{} npx markdown-link-check --quiet '{}' + + - name: Remark lint + run: | + npx remark docs -qf + + - name: Setup Python + uses: actions/setup-python@v5 + with: + python-version: ${{ env.PYTHON_VERSION }} + + - name: Install documentation dependencies + run: | + python -m pip install --upgrade pip + python -m pip install markdown pygments + + - name: Render documentation preview bundle + run: | + python scripts/render_docs.py --source docs --output "$DOCS_OUTPUT_DIR" --clean + + - name: Upload documentation preview + if: always() + uses: actions/upload-artifact@v4 + with: + name: feedser-docs-preview + path: ${{ env.DOCS_OUTPUT_DIR }} + retention-days: 7 diff --git a/.gitea/workflows/promote.yml b/.gitea/workflows/promote.yml new file mode 100644 index 00000000..21b9b996 --- /dev/null +++ b/.gitea/workflows/promote.yml @@ -0,0 +1,206 @@ +# .gitea/workflows/promote.yml +# Manual promotion workflow to copy staged artefacts to production + +name: Promote Feedser (Manual) + +on: + workflow_dispatch: + inputs: + include_docs: + description: 'Also promote the generated documentation bundle' + required: false + default: 'true' + type: boolean + tag: + description: 'Optional build identifier to record in the summary' + required: false + default: 'latest' + type: string + +jobs: + promote: + runs-on: ubuntu-22.04 + environment: production + steps: + - name: Checkout repository + uses: actions/checkout@v4 + + - name: Resolve staging credentials + id: staging + run: | + missing=() + + host="${{ secrets.STAGING_DEPLOYMENT_HOST }}" + if [ -z "$host" ]; then host="${{ vars.STAGING_DEPLOYMENT_HOST }}"; fi + if [ -z "$host" ]; then host="${{ secrets.DEPLOYMENT_HOST }}"; fi + if [ -z "$host" ]; then host="${{ vars.DEPLOYMENT_HOST }}"; fi + if [ -z "$host" ]; then missing+=("STAGING_DEPLOYMENT_HOST"); fi + + user="${{ secrets.STAGING_DEPLOYMENT_USERNAME }}" + if [ -z "$user" ]; then user="${{ vars.STAGING_DEPLOYMENT_USERNAME }}"; fi + if [ -z "$user" ]; then user="${{ secrets.DEPLOYMENT_USERNAME }}"; fi + if [ -z "$user" ]; then user="${{ vars.DEPLOYMENT_USERNAME }}"; fi + if [ -z "$user" ]; then missing+=("STAGING_DEPLOYMENT_USERNAME"); fi + + path="${{ secrets.STAGING_DEPLOYMENT_PATH }}" + if [ -z "$path" ]; then path="${{ vars.STAGING_DEPLOYMENT_PATH }}"; fi + if [ -z "$path" ]; then missing+=("STAGING_DEPLOYMENT_PATH") + fi + + docs_path="${{ secrets.STAGING_DOCS_PATH }}" + if [ -z "$docs_path" ]; then docs_path="${{ vars.STAGING_DOCS_PATH }}"; fi + + key="${{ secrets.STAGING_DEPLOYMENT_KEY }}" + if [ -z "$key" ]; then key="${{ secrets.DEPLOYMENT_KEY }}"; fi + if [ -z "$key" ]; then key="${{ vars.STAGING_DEPLOYMENT_KEY }}"; fi + if [ -z "$key" ]; then key="${{ vars.DEPLOYMENT_KEY }}"; fi + if [ -z "$key" ]; then missing+=("STAGING_DEPLOYMENT_KEY"); fi + + if [ ${#missing[@]} -gt 0 ]; then + echo "❌ Missing staging configuration: ${missing[*]}" + exit 1 + fi + + key_file="$RUNNER_TEMP/staging_key" + printf '%s\n' "$key" > "$key_file" + chmod 600 "$key_file" + + echo "host=$host" >> $GITHUB_OUTPUT + echo "user=$user" >> $GITHUB_OUTPUT + echo "path=$path" >> $GITHUB_OUTPUT + echo "docs-path=$docs_path" >> $GITHUB_OUTPUT + echo "key-file=$key_file" >> $GITHUB_OUTPUT + + - name: Resolve production credentials + id: production + run: | + missing=() + + host="${{ secrets.PRODUCTION_DEPLOYMENT_HOST }}" + if [ -z "$host" ]; then host="${{ vars.PRODUCTION_DEPLOYMENT_HOST }}"; fi + if [ -z "$host" ]; then host="${{ secrets.DEPLOYMENT_HOST }}"; fi + if [ -z "$host" ]; then host="${{ vars.DEPLOYMENT_HOST }}"; fi + if [ -z "$host" ]; then missing+=("PRODUCTION_DEPLOYMENT_HOST"); fi + + user="${{ secrets.PRODUCTION_DEPLOYMENT_USERNAME }}" + if [ -z "$user" ]; then user="${{ vars.PRODUCTION_DEPLOYMENT_USERNAME }}"; fi + if [ -z "$user" ]; then user="${{ secrets.DEPLOYMENT_USERNAME }}"; fi + if [ -z "$user" ]; then user="${{ vars.DEPLOYMENT_USERNAME }}"; fi + if [ -z "$user" ]; then missing+=("PRODUCTION_DEPLOYMENT_USERNAME"); fi + + path="${{ secrets.PRODUCTION_DEPLOYMENT_PATH }}" + if [ -z "$path" ]; then path="${{ vars.PRODUCTION_DEPLOYMENT_PATH }}"; fi + if [ -z "$path" ]; then missing+=("PRODUCTION_DEPLOYMENT_PATH") + fi + + docs_path="${{ secrets.PRODUCTION_DOCS_PATH }}" + if [ -z "$docs_path" ]; then docs_path="${{ vars.PRODUCTION_DOCS_PATH }}"; fi + + key="${{ secrets.PRODUCTION_DEPLOYMENT_KEY }}" + if [ -z "$key" ]; then key="${{ secrets.DEPLOYMENT_KEY }}"; fi + if [ -z "$key" ]; then key="${{ vars.PRODUCTION_DEPLOYMENT_KEY }}"; fi + if [ -z "$key" ]; then key="${{ vars.DEPLOYMENT_KEY }}"; fi + if [ -z "$key" ]; then missing+=("PRODUCTION_DEPLOYMENT_KEY"); fi + + if [ ${#missing[@]} -gt 0 ]; then + echo "❌ Missing production configuration: ${missing[*]}" + exit 1 + fi + + key_file="$RUNNER_TEMP/production_key" + printf '%s\n' "$key" > "$key_file" + chmod 600 "$key_file" + + echo "host=$host" >> $GITHUB_OUTPUT + echo "user=$user" >> $GITHUB_OUTPUT + echo "path=$path" >> $GITHUB_OUTPUT + echo "docs-path=$docs_path" >> $GITHUB_OUTPUT + echo "key-file=$key_file" >> $GITHUB_OUTPUT + + - name: Install rsync + run: | + if command -v rsync >/dev/null 2>&1; then + exit 0 + fi + CACHE_DIR="${CI_CACHE_ROOT:-/tmp}/apt" + mkdir -p "$CACHE_DIR" + KEY="rsync-$(lsb_release -rs 2>/dev/null || echo unknown)" + DEB_DIR="$CACHE_DIR/$KEY" + mkdir -p "$DEB_DIR" + if ls "$DEB_DIR"/rsync*.deb >/dev/null 2>&1; then + apt-get update + apt-get install -y --no-install-recommends "$DEB_DIR"/libpopt0*.deb "$DEB_DIR"/rsync*.deb + else + apt-get update + apt-get download rsync libpopt0 + mv rsync*.deb libpopt0*.deb "$DEB_DIR"/ + dpkg -i "$DEB_DIR"/libpopt0*.deb "$DEB_DIR"/rsync*.deb || apt-get install -f -y + fi + + - name: Fetch staging artefacts + id: fetch + run: | + staging_root="${{ runner.temp }}/staging" + mkdir -p "$staging_root/service" "$staging_root/docs" + + echo "📥 Copying service bundle from staging" + rsync -az --delete \ + -e "ssh -i ${{ steps.staging.outputs['key-file'] }} -o StrictHostKeyChecking=no" \ + "${{ steps.staging.outputs.user }}@${{ steps.staging.outputs.host }}:${{ steps.staging.outputs.path }}/" \ + "$staging_root/service/" + + if [ "${{ github.event.inputs.include_docs }}" = "true" ] && [ -n "${{ steps.staging.outputs['docs-path'] }}" ]; then + echo "📥 Copying documentation bundle from staging" + rsync -az --delete \ + -e "ssh -i ${{ steps.staging.outputs['key-file'] }} -o StrictHostKeyChecking=no" \ + "${{ steps.staging.outputs.user }}@${{ steps.staging.outputs.host }}:${{ steps.staging.outputs['docs-path'] }}/" \ + "$staging_root/docs/" + else + echo "ℹ️ Documentation promotion skipped" + fi + + echo "service-dir=$staging_root/service" >> $GITHUB_OUTPUT + echo "docs-dir=$staging_root/docs" >> $GITHUB_OUTPUT + + - name: Backup production service content + run: | + ssh -o StrictHostKeyChecking=no -i "${{ steps.production.outputs['key-file'] }}" \ + "${{ steps.production.outputs.user }}@${{ steps.production.outputs.host }}" \ + "set -e; TARGET='${{ steps.production.outputs.path }}'; \ + if [ -d \"$TARGET\" ]; then \ + parent=\$(dirname \"$TARGET\"); \ + base=\$(basename \"$TARGET\"); \ + backup=\"\$parent/\${base}.backup.\$(date +%Y%m%d_%H%M%S)\"; \ + mkdir -p \"\$backup\"; \ + rsync -a --delete \"$TARGET/\" \"\$backup/\"; \ + ls -dt \"\$parent/\${base}.backup.*\" 2>/dev/null | tail -n +6 | xargs rm -rf || true; \ + echo 'Backup created at ' \"\$backup\"; \ + else \ + echo 'Production service path missing; skipping backup'; \ + fi" + + - name: Publish service to production + run: | + rsync -az --delete \ + -e "ssh -i ${{ steps.production.outputs['key-file'] }} -o StrictHostKeyChecking=no" \ + "${{ steps.fetch.outputs['service-dir'] }}/" \ + "${{ steps.production.outputs.user }}@${{ steps.production.outputs.host }}:${{ steps.production.outputs.path }}/" + + - name: Promote documentation bundle + if: github.event.inputs.include_docs == 'true' && steps.production.outputs['docs-path'] != '' + run: | + rsync -az --delete \ + -e "ssh -i ${{ steps.production.outputs['key-file'] }} -o StrictHostKeyChecking=no" \ + "${{ steps.fetch.outputs['docs-dir'] }}/" \ + "${{ steps.production.outputs.user }}@${{ steps.production.outputs.host }}:${{ steps.production.outputs['docs-path'] }}/" + + - name: Promotion summary + run: | + echo "✅ Promotion completed" + echo " Tag: ${{ github.event.inputs.tag }}" + echo " Service: ${{ steps.staging.outputs.host }} → ${{ steps.production.outputs.host }}" + if [ "${{ github.event.inputs.include_docs }}" = "true" ]; then + echo " Docs: included" + else + echo " Docs: skipped" + fi diff --git a/.gitignore b/.gitignore new file mode 100644 index 00000000..b4bca2c2 --- /dev/null +++ b/.gitignore @@ -0,0 +1,21 @@ +# Build outputs +bin/ +obj/ +*.pdb +*.dll + +# IDE state +.vs/ +*.user +*.suo +*.userprefs + +# Rider/VSCode +.idea/ +.vscode/ + +# Packages and logs +*.log +TestResults/ + +.dotnet \ No newline at end of file diff --git a/AGENTS.md b/AGENTS.md new file mode 100644 index 00000000..5a50c0f9 --- /dev/null +++ b/AGENTS.md @@ -0,0 +1,125 @@ +# 1) What is StellaOps? + +**StellaOps** an open, sovereign, modular container-security toolkit built for high-speed, offline operation, released under AGPL-3.0-or-later. + +It follows an SBOM-first model—analyzing each container layer or ingesting existing CycloneDX/SPDX SBOMs, then enriching them with vulnerability, licence, secret-leak, and misconfiguration data to produce cryptographically signed reports. + +Vulnerability detection maps OS and language dependencies to sources such as NVD, GHSA, OSV, ENISA. +Secrets sweep flags exposed credentials or keys in files or environment variables. +Licence audit identifies potential conflicts, especially copyleft obligations. +Misconfiguration checks detect unsafe Dockerfile patterns (root user, latest tags, permissive modes). +Provenance features include in-toto/SLSA attestations signed with cosign for supply-chain trust. + +| Guiding principle | What it means for Feedser | +|-------------------|---------------------------| +| **SBOM-first ingest** | Prefer signed SBOMs or reproducible layer diffs before falling back to raw scraping; connectors treat source docs as provenance, never as mutable truth. | +| **Deterministic outputs** | Same inputs yield identical canonical advisories and exported JSON/Trivy DB artefacts; merge hashes and export manifests are reproducible across machines. | +| **Restart-time plug-ins only** | Connector/exporter plug-ins load at service start, keeping runtime sandboxing simple and avoiding hot-patch attack surface. | +| **Sovereign/offline-first** | No mandatory outbound calls beyond allow-listed advisories; Offline Kit bundles Mongo snapshots and exporter artefacts for air-gapped installs. | +| **Operational transparency** | Every stage logs structured events (fetch, parse, merge, export) with correlation IDs so parallel agents can debug without shared state. | + +Performance: warm scans < 5 s, cold scans < 30 s on a 4 vCPU runner. +Deployment: entirely SaaS-free, suitable for air-gapped or on-prem use through its Offline Kit. +Policy: anonymous users → 33 scans/day; verified → 333 /day; nearing 90 % quota triggers throttling but never full blocks. + +More documention is available ./docs/*.md files. Read `docs/README.md` to gather information about the available documentation. You could inquiry specific documents as your work requires it + +--- + +# 3) Practices + +## 3.1) Naming +All modules are .NET projects based on .NET 10 (preview). Exclussion is the UI. It is based on Angular +All modules are contained by one or more projects. Each project goes in its dedicated folder. Each project starts with StellaOps.. In case it is common for for all StellaOps modules it is library or plugin and it is named StellaOps.. + +## 3.2) Key technologies & integrations + +- **Runtime**: .NET 10 (`net10.0`) preview SDK; C# latest preview features. +- **Data**: MongoDB (canonical store and job/export state). +- **Observability**: structured logs, counters, and (optional) OpenTelemetry traces. +- **Ops posture**: offline‑first, allowlist for remote hosts, strict schema validation, gated LLM fallback (only where explicitly configured). + +# 4) Modules +StellaOps is contained by different modules installable via docker containers +- Feedser. Responsible for aggregation and delivery of vulnerability database +- Cli. Command line tool to unlock full potential - request database operations, install scanner, request scan, configure backend +- Backend. Configures and Manages scans +- UI. UI to access the backend (and scanners) +- Agent. Installable daemon that does the scanning +- Zastava. Realtime monitor for allowed (verified) installations. + +## 4.1) Feedser +It is webservice based module that is responsible for aggregating vulnerabilities information from various sources, parsing and normalizing them into a canonical shape, merging and deduplicating the results in one place, with export capabilities to Json and TrivyDb. It supports init and resume for all of the sources, parse/normalize and merge/deduplication operations, plus export. Export supports delta exports—similarly to full and incremential database backups. + +### 4.1.1) Usage +It supports operations to be started by cmd line: +# stella db [fetch|merge|export] [init|resume ] +or +api available on https://db.stella-ops.org + +### 4.1.2) Data flow (end‑to‑end) + +1. **Fetch**: connectors request source windows with retries/backoff, persist raw documents with SHA256/ETag metadata. +2. **Parse & Normalize**: validate to DTOs (schema-checked), quarantine failures, normalize to canonical advisories (aliases, affected ranges with NEVRA/EVR/SemVer, references, provenance). +3. **Merge & Deduplicate**: enforce precedence, build/maintain alias graphs, compute deterministic hashes, and eliminate duplicates before persisting to MongoDB. +4. **Export**: JSON tree and/or Trivy DB; package and (optionally) push; write export state. + +### 4.1.3) Architecture +For more information of the architecture see `./docs/ARCHITECTURE_FEEDSER.md`. + +--- + +### 4.1.4) Glossary (quick) + +- **OVAL** — Vendor/distro security definition format; authoritative for OS packages. +- **NEVRA / EVR** — RPM and Debian version semantics for OS packages. +- **PURL / SemVer** — Coordinates and version semantics for OSS ecosystems. +- **KEV** — Known Exploited Vulnerabilities (flag only). + +--- +# 5) Your role as StellaOps contributor + +You acting as information technology engineer that will take different type of roles in goal achieving StellaOps production implementation +In order you to work - you have to be supplied with directory that contains `AGENTS.md`,`TASKS.md` files. There will you have more information about the role you have, the scope of your work and the tasks you will have. + +Boundaries: +- You operate only in the working directories I gave you, unless there is dependencies that makes you to work on dependency in shared directory. Then you ask for confirmation. + +You main characteristics: +- Keep endpoints small, deterministic, and cancellation-aware. +- Improve logs/metrics as per tasks. +- Update `TASKS.md` when moving tasks forward. +- When you are done with all task you state explicitly you are done. +- Impersonate the role described on working directory `AGENTS.md` you will read, if role is not available - take role of the CTO of the StellaOps in early stages. +- You always strive for best practices +- You always strive for re-usability +- When in doubt of design decision - you ask then act +- You are autonomus - meaning that you will work for long time alone and achieve maximum without stopping for stupid questions +- You operate on the same directory where other agents will work. In case you need to work on directory that is dependency on provided `AGENTS.md`,`TASKS.md` files you have to ask for confirmation first. + +## 5.1) Type of contributions + +- **BE‑Base (Platform & Pipeline)** + Owns DI, plugin host, job scheduler/coordinator, configuration binding, minimal API endpoints, and Mongo bootstrapping. +- **BE‑Conn‑X (Connectors)** + One agent per source family (NVD, Red Hat, Ubuntu, Debian, SUSE, GHSA, OSV, PSIRTs, CERTs, KEV, ICS). Implements fetch/parse/map with incremental watermarks. +- **BE‑Merge (Canonical Merge & Dedupe)** + Identity graph, precedence policies, canonical JSON serializer, and deterministic hashing (`merge_event`). +- **BE‑Export (JSON & Trivy DB)** + Deterministic export trees, Trivy DB packaging, optional ORAS push, and offline bundle. +- **QA (Validation & Observability)** + Schema tests, fixture goldens, determinism checks, metrics/logs/traces, e2e reproducibility runs. +- **DevEx/Docs** + Maintains this agent framework, templates, and per‑directory guides; assists parallelization and reviews. + + +## 5.2) Work-in-parallel rules (important) + +- **Directory ownership**: Each agent works **only inside its module directory**. Cross‑module edits require a brief handshake in issues/PR description. +- **Scoping**: Use each module’s `AGENTS.md` and `TASKS.md` to plan; autonomous agents must read `src/AGENTS.md` and the module docs before acting. +- **Determinism**: Sort keys, normalize timestamps to UTC ISO‑8601, avoid non‑deterministic data in exports and tests. +- **Status tracking**: Update your module’s `TASKS.md` as you progress (TODO → DOING → DONE/BLOCKED). Before starting of actual work - ensure you have set the task to DOING. When complete or stop update the status in corresponding TASKS.md or in ./SPRINTS.md file. +- **Tests**: Add/extend fixtures and unit tests per change; never regress determinism or precedence. +- **Test layout**: Use module-specific projects in `StellaOps.Feedser..Tests`; shared fixtures/harnesses live in `StellaOps.Feedser.Testing`. + +--- diff --git a/LICENSE b/LICENSE new file mode 100755 index 00000000..0cabe4cc --- /dev/null +++ b/LICENSE @@ -0,0 +1,235 @@ +GNU AFFERO GENERAL PUBLIC LICENSE +Version 3, 19 November 2007 + +Copyright (C) 2007 Free Software Foundation, Inc. + +Everyone is permitted to copy and distribute verbatim copies of this license document, but changing it is not allowed. + + Preamble + +The GNU Affero General Public License is a free, copyleft license for software and other kinds of works, specifically designed to ensure cooperation with the community in the case of network server software. + +The licenses for most software and other practical works are designed to take away your freedom to share and change the works. By contrast, our General Public Licenses are intended to guarantee your freedom to share and change all versions of a program--to make sure it remains free software for all its users. + +When we speak of free software, we are referring to freedom, not price. Our General Public Licenses are designed to make sure that you have the freedom to distribute copies of free software (and charge for them if you wish), that you receive source code or can get it if you want it, that you can change the software or use pieces of it in new free programs, and that you know you can do these things. + +Developers that use our General Public Licenses protect your rights with two steps: (1) assert copyright on the software, and (2) offer you this License which gives you legal permission to copy, distribute and/or modify the software. + +A secondary benefit of defending all users' freedom is that improvements made in alternate versions of the program, if they receive widespread use, become available for other developers to incorporate. Many developers of free software are heartened and encouraged by the resulting cooperation. However, in the case of software used on network servers, this result may fail to come about. The GNU General Public License permits making a modified version and letting the public access it on a server without ever releasing its source code to the public. + +The GNU Affero General Public License is designed specifically to ensure that, in such cases, the modified source code becomes available to the community. It requires the operator of a network server to provide the source code of the modified version running there to the users of that server. Therefore, public use of a modified version, on a publicly accessible server, gives the public access to the source code of the modified version. + +An older license, called the Affero General Public License and published by Affero, was designed to accomplish similar goals. This is a different license, not a version of the Affero GPL, but Affero has released a new version of the Affero GPL which permits relicensing under this license. + +The precise terms and conditions for copying, distribution and modification follow. + + TERMS AND CONDITIONS + +0. Definitions. + +"This License" refers to version 3 of the GNU Affero General Public License. + +"Copyright" also means copyright-like laws that apply to other kinds of works, such as semiconductor masks. + +"The Program" refers to any copyrightable work licensed under this License. Each licensee is addressed as "you". "Licensees" and "recipients" may be individuals or organizations. + +To "modify" a work means to copy from or adapt all or part of the work in a fashion requiring copyright permission, other than the making of an exact copy. The resulting work is called a "modified version" of the earlier work or a work "based on" the earlier work. + +A "covered work" means either the unmodified Program or a work based on the Program. + +To "propagate" a work means to do anything with it that, without permission, would make you directly or secondarily liable for infringement under applicable copyright law, except executing it on a computer or modifying a private copy. Propagation includes copying, distribution (with or without modification), making available to the public, and in some countries other activities as well. + +To "convey" a work means any kind of propagation that enables other parties to make or receive copies. Mere interaction with a user through a computer network, with no transfer of a copy, is not conveying. + +An interactive user interface displays "Appropriate Legal Notices" to the extent that it includes a convenient and prominently visible feature that (1) displays an appropriate copyright notice, and (2) tells the user that there is no warranty for the work (except to the extent that warranties are provided), that licensees may convey the work under this License, and how to view a copy of this License. If the interface presents a list of user commands or options, such as a menu, a prominent item in the list meets this criterion. + +1. Source Code. +The "source code" for a work means the preferred form of the work for making modifications to it. "Object code" means any non-source form of a work. + +A "Standard Interface" means an interface that either is an official standard defined by a recognized standards body, or, in the case of interfaces specified for a particular programming language, one that is widely used among developers working in that language. + +The "System Libraries" of an executable work include anything, other than the work as a whole, that (a) is included in the normal form of packaging a Major Component, but which is not part of that Major Component, and (b) serves only to enable use of the work with that Major Component, or to implement a Standard Interface for which an implementation is available to the public in source code form. A "Major Component", in this context, means a major essential component (kernel, window system, and so on) of the specific operating system (if any) on which the executable work runs, or a compiler used to produce the work, or an object code interpreter used to run it. + +The "Corresponding Source" for a work in object code form means all the source code needed to generate, install, and (for an executable work) run the object code and to modify the work, including scripts to control those activities. However, it does not include the work's System Libraries, or general-purpose tools or generally available free programs which are used unmodified in performing those activities but which are not part of the work. For example, Corresponding Source includes interface definition files associated with source files for the work, and the source code for shared libraries and dynamically linked subprograms that the work is specifically designed to require, such as by intimate data communication or control flow between those +subprograms and other parts of the work. + +The Corresponding Source need not include anything that users can regenerate automatically from other parts of the Corresponding Source. + +The Corresponding Source for a work in source code form is that same work. + +2. Basic Permissions. +All rights granted under this License are granted for the term of copyright on the Program, and are irrevocable provided the stated conditions are met. This License explicitly affirms your unlimited permission to run the unmodified Program. The output from running a covered work is covered by this License only if the output, given its content, constitutes a covered work. This License acknowledges your rights of fair use or other equivalent, as provided by copyright law. + +You may make, run and propagate covered works that you do not convey, without conditions so long as your license otherwise remains in force. You may convey covered works to others for the sole purpose of having them make modifications exclusively for you, or provide you with facilities for running those works, provided that you comply with the terms of this License in conveying all material for which you do not control copyright. Those thus making or running the covered works for you must do so exclusively on your behalf, under your direction and control, on terms that prohibit them from making any copies of your copyrighted material outside their relationship with you. + +Conveying under any other circumstances is permitted solely under the conditions stated below. Sublicensing is not allowed; section 10 makes it unnecessary. + +3. Protecting Users' Legal Rights From Anti-Circumvention Law. +No covered work shall be deemed part of an effective technological measure under any applicable law fulfilling obligations under article 11 of the WIPO copyright treaty adopted on 20 December 1996, or similar laws prohibiting or restricting circumvention of such measures. + +When you convey a covered work, you waive any legal power to forbid circumvention of technological measures to the extent such circumvention is effected by exercising rights under this License with respect to the covered work, and you disclaim any intention to limit operation or modification of the work as a means of enforcing, against the work's users, your or third parties' legal rights to forbid circumvention of technological measures. + +4. Conveying Verbatim Copies. +You may convey verbatim copies of the Program's source code as you receive it, in any medium, provided that you conspicuously and appropriately publish on each copy an appropriate copyright notice; keep intact all notices stating that this License and any non-permissive terms added in accord with section 7 apply to the code; keep intact all notices of the absence of any warranty; and give all recipients a copy of this License along with the Program. + +You may charge any price or no price for each copy that you convey, and you may offer support or warranty protection for a fee. + +5. Conveying Modified Source Versions. +You may convey a work based on the Program, or the modifications to produce it from the Program, in the form of source code under the terms of section 4, provided that you also meet all of these conditions: + + a) The work must carry prominent notices stating that you modified it, and giving a relevant date. + + b) The work must carry prominent notices stating that it is released under this License and any conditions added under section 7. This requirement modifies the requirement in section 4 to "keep intact all notices". + + c) You must license the entire work, as a whole, under this License to anyone who comes into possession of a copy. This License will therefore apply, along with any applicable section 7 additional terms, to the whole of the work, and all its parts, regardless of how they are packaged. This License gives no permission to license the work in any other way, but it does not invalidate such permission if you have separately received it. + + d) If the work has interactive user interfaces, each must display Appropriate Legal Notices; however, if the Program has interactive interfaces that do not display Appropriate Legal Notices, your work need not make them do so. + +A compilation of a covered work with other separate and independent works, which are not by their nature extensions of the covered work, and which are not combined with it such as to form a larger program, in or on a volume of a storage or distribution medium, is called an "aggregate" if the compilation and its resulting copyright are not used to limit the access or legal rights of the compilation's users beyond what the individual works permit. Inclusion of a covered work in an aggregate does not cause this License to apply to the other parts of the aggregate. + +6. Conveying Non-Source Forms. +You may convey a covered work in object code form under the terms of sections 4 and 5, provided that you also convey the machine-readable Corresponding Source under the terms of this License, in one of these ways: + + a) Convey the object code in, or embodied in, a physical product (including a physical distribution medium), accompanied by the Corresponding Source fixed on a durable physical medium customarily used for software interchange. + + b) Convey the object code in, or embodied in, a physical product (including a physical distribution medium), accompanied by a written offer, valid for at least three years and valid for as long as you offer spare parts or customer support for that product model, to give anyone who possesses the object code either (1) a copy of the Corresponding Source for all the software in the product that is covered by this License, on a durable physical medium customarily used for software interchange, for a price no more than your reasonable cost of physically performing this conveying of source, or (2) access to copy the Corresponding Source from a network server at no charge. + + c) Convey individual copies of the object code with a copy of the written offer to provide the Corresponding Source. This alternative is allowed only occasionally and noncommercially, and only if you received the object code with such an offer, in accord with subsection 6b. + + d) Convey the object code by offering access from a designated place (gratis or for a charge), and offer equivalent access to the Corresponding Source in the same way through the same place at no further charge. You need not require recipients to copy the Corresponding Source along with the object code. If the place to copy the object code is a network server, the Corresponding Source may be on a different server (operated by you or a third party) that supports equivalent copying facilities, provided you maintain clear directions next to the object code saying where to find the Corresponding Source. Regardless of what server hosts the Corresponding Source, you remain obligated to ensure that it is available for as long as needed to satisfy these requirements. + + e) Convey the object code using peer-to-peer transmission, provided you inform other peers where the object code and Corresponding Source of the work are being offered to the general public at no charge under subsection 6d. + +A separable portion of the object code, whose source code is excluded from the Corresponding Source as a System Library, need not be included in conveying the object code work. + +A "User Product" is either (1) a "consumer product", which means any tangible personal property which is normally used for personal, family, or household purposes, or (2) anything designed or sold for incorporation into a dwelling. In determining whether a product is a consumer product, doubtful cases shall be resolved in favor of coverage. For a particular product received by a particular user, "normally used" refers to a typical or common use of that class of product, regardless of the status of the particular user or of the way in which the particular user actually uses, or expects or is expected to use, the product. A product is a consumer product regardless of whether the product has substantial commercial, industrial or non-consumer uses, unless such uses represent the only significant mode of use of the product. + +"Installation Information" for a User Product means any methods, procedures, authorization keys, or other information required to install and execute modified versions of a covered work in that User Product from a modified version of its Corresponding Source. The information must suffice to ensure that the continued functioning of the modified object code is in no case prevented or interfered with solely because modification has been made. + +If you convey an object code work under this section in, or with, or specifically for use in, a User Product, and the conveying occurs as part of a transaction in which the right of possession and use of the User Product is transferred to the recipient in perpetuity or for a fixed term (regardless of how the transaction is characterized), the Corresponding Source conveyed under this section must be accompanied by the Installation Information. But this requirement does not apply if neither you nor any third party retains the ability to install modified object code on the User Product (for example, the work has been installed in ROM). + +The requirement to provide Installation Information does not include a requirement to continue to provide support service, warranty, or updates for a work that has been modified or installed by the recipient, or for the User Product in which it has been modified or installed. Access to a network may be denied when the modification itself materially and adversely affects the operation of the network or violates the rules and protocols for communication across the network. + +Corresponding Source conveyed, and Installation Information provided, in accord with this section must be in a format that is publicly documented (and with an implementation available to the public in source code form), and must require no special password or key for unpacking, reading or copying. + +7. Additional Terms. +"Additional permissions" are terms that supplement the terms of this License by making exceptions from one or more of its conditions. Additional permissions that are applicable to the entire Program shall be treated as though they were included in this License, to the extent that they are valid under applicable law. If additional permissions apply only to part of the Program, that part may be used separately under those permissions, but the entire Program remains governed by this License without regard to the additional permissions. + +When you convey a copy of a covered work, you may at your option remove any additional permissions from that copy, or from any part of it. (Additional permissions may be written to require their own removal in certain cases when you modify the work.) You may place additional permissions on material, added by you to a covered work, for which you have or can give appropriate copyright permission. + +Notwithstanding any other provision of this License, for material you add to a covered work, you may (if authorized by the copyright holders of that material) supplement the terms of this License with terms: + + a) Disclaiming warranty or limiting liability differently from the terms of sections 15 and 16 of this License; or + + b) Requiring preservation of specified reasonable legal notices or author attributions in that material or in the Appropriate Legal Notices displayed by works containing it; or + + c) Prohibiting misrepresentation of the origin of that material, or requiring that modified versions of such material be marked in reasonable ways as different from the original version; or + + d) Limiting the use for publicity purposes of names of licensors or authors of the material; or + + e) Declining to grant rights under trademark law for use of some trade names, trademarks, or service marks; or + + f) Requiring indemnification of licensors and authors of that material by anyone who conveys the material (or modified versions of it) with contractual assumptions of liability to the recipient, for any liability that these contractual assumptions directly impose on those licensors and authors. + +All other non-permissive additional terms are considered "further restrictions" within the meaning of section 10. If the Program as you received it, or any part of it, contains a notice stating that it is governed by this License along with a term that is a further restriction, you may remove that term. If a license document contains a further restriction but permits relicensing or conveying under this License, you may add to a covered work material governed by the terms of that license document, provided that the further restriction does not survive such relicensing or conveying. + +If you add terms to a covered work in accord with this section, you must place, in the relevant source files, a statement of the additional terms that apply to those files, or a notice indicating where to find the applicable terms. + +Additional terms, permissive or non-permissive, may be stated in the form of a separately written license, or stated as exceptions; the above requirements apply either way. + +8. Termination. + +You may not propagate or modify a covered work except as expressly provided under this License. Any attempt otherwise to propagate or modify it is void, and will automatically terminate your rights under this License (including any patent licenses granted under the third paragraph of section 11). + +However, if you cease all violation of this License, then your license from a particular copyright holder is reinstated (a) provisionally, unless and until the copyright holder explicitly and finally terminates your license, and (b) permanently, if the copyright holder fails to notify you of the violation by some reasonable means prior to 60 days after the cessation. + +Moreover, your license from a particular copyright holder is reinstated permanently if the copyright holder notifies you of the violation by some reasonable means, this is the first time you have received notice of violation of this License (for any work) from that copyright holder, and you cure the violation prior to 30 days after your receipt of the notice. + +Termination of your rights under this section does not terminate the licenses of parties who have received copies or rights from you under this License. If your rights have been terminated and not permanently reinstated, you do not qualify to receive new licenses for the same material under section 10. + +9. Acceptance Not Required for Having Copies. + +You are not required to accept this License in order to receive or run a copy of the Program. Ancillary propagation of a covered work occurring solely as a consequence of using peer-to-peer transmission to receive a copy likewise does not require acceptance. However, nothing other than this License grants you permission to propagate or modify any covered work. These actions infringe copyright if you do not accept this License. Therefore, by modifying or propagating a covered work, you indicate your acceptance of this License to do so. + +10. Automatic Licensing of Downstream Recipients. + +Each time you convey a covered work, the recipient automatically receives a license from the original licensors, to run, modify and propagate that work, subject to this License. You are not responsible for enforcing compliance by third parties with this License. + +An "entity transaction" is a transaction transferring control of an organization, or substantially all assets of one, or subdividing an organization, or merging organizations. If propagation of a covered work results from an entity transaction, each party to that transaction who receives a copy of the work also receives whatever licenses to the work the party's predecessor in interest had or could give under the previous paragraph, plus a right to possession of the Corresponding Source of the work from the predecessor in interest, if the predecessor has it or can get it with reasonable efforts. + +You may not impose any further restrictions on the exercise of the rights granted or affirmed under this License. For example, you may not impose a license fee, royalty, or other charge for exercise of rights granted under this License, and you may not initiate litigation (including a cross-claim or counterclaim in a lawsuit) alleging that any patent claim is infringed by making, using, selling, offering for sale, or importing the Program or any portion of it. + +11. Patents. + +A "contributor" is a copyright holder who authorizes use under this License of the Program or a work on which the Program is based. The work thus licensed is called the contributor's "contributor version". + +A contributor's "essential patent claims" are all patent claims owned or controlled by the contributor, whether already acquired or hereafter acquired, that would be infringed by some manner, permitted by this License, of making, using, or selling its contributor version, but do not include claims that would be infringed only as a consequence of further modification of the contributor version. For purposes of this definition, "control" includes the right to grant patent sublicenses in a manner consistent with the requirements of this License. + +Each contributor grants you a non-exclusive, worldwide, royalty-free patent license under the contributor's essential patent claims, to make, use, sell, offer for sale, import and otherwise run, modify and propagate the contents of its contributor version. + +In the following three paragraphs, a "patent license" is any express agreement or commitment, however denominated, not to enforce a patent (such as an express permission to practice a patent or covenant not to sue for patent infringement). To "grant" such a patent license to a party means to make such an agreement or commitment not to enforce a patent against the party. + +If you convey a covered work, knowingly relying on a patent license, and the Corresponding Source of the work is not available for anyone to copy, free of charge and under the terms of this License, through a publicly available network server or other readily accessible means, then you must either (1) cause the Corresponding Source to be so available, or (2) arrange to deprive yourself of the benefit of the patent license for this particular work, or (3) arrange, in a manner consistent with the requirements of this License, to extend the patent +license to downstream recipients. "Knowingly relying" means you have actual knowledge that, but for the patent license, your conveying the covered work in a country, or your recipient's use of the covered work in a country, would infringe one or more identifiable patents in that country that you have reason to believe are valid. + +If, pursuant to or in connection with a single transaction or arrangement, you convey, or propagate by procuring conveyance of, a covered work, and grant a patent license to some of the parties receiving the covered work authorizing them to use, propagate, modify or convey a specific copy of the covered work, then the patent license you grant is automatically extended to all recipients of the covered work and works based on it. + +A patent license is "discriminatory" if it does not include within the scope of its coverage, prohibits the exercise of, or is conditioned on the non-exercise of one or more of the rights that are specifically granted under this License. You may not convey a covered work if you are a party to an arrangement with a third party that is in the business of distributing software, under which you make payment to the third party based on the extent of your activity of conveying the work, and under which the third party grants, to any of the parties who would receive the covered work from you, a discriminatory patent license (a) in connection with copies of the covered work conveyed by you (or copies made from those copies), or (b) primarily for and in connection with specific products or compilations that contain the covered work, unless you entered into that arrangement, or that patent license was granted, prior to 28 March 2007. + +Nothing in this License shall be construed as excluding or limiting any implied license or other defenses to infringement that may otherwise be available to you under applicable patent law. + +12. No Surrender of Others' Freedom. + +If conditions are imposed on you (whether by court order, agreement or otherwise) that contradict the conditions of this License, they do not excuse you from the conditions of this License. If you cannot convey a covered work so as to satisfy simultaneously your obligations under this License and any other pertinent obligations, then as a consequence you may +not convey it at all. For example, if you agree to terms that obligate you to collect a royalty for further conveying from those to whom you convey the Program, the only way you could satisfy both those terms and this License would be to refrain entirely from conveying the Program. + +13. Remote Network Interaction; Use with the GNU General Public License. + +Notwithstanding any other provision of this License, if you modify the Program, your modified version must prominently offer all users interacting with it remotely through a computer network (if your version supports such interaction) an opportunity to receive the Corresponding Source of your version by providing access to the Corresponding Source from a network server at no charge, through some standard or customary means of facilitating copying of software. This Corresponding Source shall include the Corresponding Source for any work covered by version 3 of the GNU General Public License that is incorporated pursuant to the following paragraph. + +Notwithstanding any other provision of this License, you have permission to link or combine any covered work with a work licensed under version 3 of the GNU General Public License into a single combined work, and to convey the resulting work. The terms of this License will continue to apply to the part which is the covered work, but the work with which it is combined will remain governed by version 3 of the GNU General Public License. + +14. Revised Versions of this License. + +The Free Software Foundation may publish revised and/or new versions of the GNU Affero General Public License from time to time. Such new versions will be similar in spirit to the present version, but may differ in detail to address new problems or concerns. + +Each version is given a distinguishing version number. If the Program specifies that a certain numbered version of the GNU Affero General Public License "or any later version" applies to it, you have the option of following the terms and conditions either of that numbered version or of any later version published by the Free Software Foundation. If the Program does not specify a version number of the GNU Affero General Public License, you may choose any version ever published by the Free Software Foundation. + +If the Program specifies that a proxy can decide which future versions of the GNU Affero General Public License can be used, that proxy's public statement of acceptance of a version permanently authorizes you to choose that version for the Program. + +Later license versions may give you additional or different permissions. However, no additional obligations are imposed on any author or copyright holder as a result of your choosing to follow a later version. + +15. Disclaimer of Warranty. + +THERE IS NO WARRANTY FOR THE PROGRAM, TO THE EXTENT PERMITTED BY APPLICABLE LAW. EXCEPT WHEN OTHERWISE STATED IN WRITING THE COPYRIGHT HOLDERS AND/OR OTHER PARTIES PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY OF ANY KIND, EITHER EXPRESSED OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE. THE ENTIRE RISK AS TO THE QUALITY AND PERFORMANCE OF THE PROGRAM IS WITH YOU. SHOULD THE PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF ALL NECESSARY SERVICING, REPAIR OR CORRECTION. + +16. Limitation of Liability. + +IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MODIFIES AND/OR CONVEYS THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES, INCLUDING ANY GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING OUT OF THE USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED TO LOSS OF DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY YOU OR THIRD PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER PROGRAMS), EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE POSSIBILITY OF SUCH DAMAGES. + +17. Interpretation of Sections 15 and 16. + +If the disclaimer of warranty and limitation of liability provided above cannot be given local legal effect according to their terms, reviewing courts shall apply local law that most closely approximates an absolute waiver of all civil liability in connection with the Program, unless a warranty or assumption of liability accompanies a copy of the Program in return for a fee. + +END OF TERMS AND CONDITIONS + + How to Apply These Terms to Your New Programs + +If you develop a new program, and you want it to be of the greatest possible use to the public, the best way to achieve this is to make it free software which everyone can redistribute and change under these terms. + +To do so, attach the following notices to the program. It is safest to attach them to the start of each source file to most effectively state the exclusion of warranty; and each file should have at least the "copyright" line and a pointer to where the full notice is found. + + git.stella-ops.org + Copyright (C) 2025 stella-ops.org + + This program is free software: you can redistribute it and/or modify it under the terms of the GNU Affero General Public License as published by the Free Software Foundation, either version 3 of the License, or (at your option) any later version. + + This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero General Public License for more details. + + You should have received a copy of the GNU Affero General Public License along with this program. If not, see . + +Also add information on how to contact you by electronic and paper mail. + +If your software can interact with users remotely through a computer network, you should also make sure that it provides a way for users to get its source. For example, if your program is a web application, its interface could display a "Source" link that leads users to an archive of the code. There are many ways you could offer source, and different solutions will be better for different programs; see section 13 for the specific requirements. + +You should also get your employer (if you work as a programmer) or school, if any, to sign a "copyright disclaimer" for the program, if necessary. For more information on this, and how to apply and follow the GNU AGPL, see . diff --git a/README.md b/README.md new file mode 100755 index 00000000..c673de47 --- /dev/null +++ b/README.md @@ -0,0 +1,28 @@ +# StellaOps Feedser & CLI + +This repository hosts the StellaOps Feedser service, its plug-in ecosystem, and the +first-party CLI (`stellaops-cli`). Feedser ingests vulnerability advisories from +authoritative sources, stores them in MongoDB, and exports deterministic JSON and +Trivy DB artefacts. The CLI drives scanner distribution, scan execution, and job +control against the Feedser API. + +## Quickstart + +1. Prepare a MongoDB instance and (optionally) install `trivy-db`/`oras`. +2. Copy `etc/feedser.yaml.sample` to `etc/feedser.yaml` and update the storage + telemetry + settings. +3. Copy `etc/authority.yaml.sample` to `etc/authority.yaml`, review the issuer, token + lifetimes, and plug-in descriptors, then edit the companion manifests under + `etc/authority.plugins/*.yaml` to match your deployment. +4. Start the web service with `dotnet run --project src/StellaOps.Feedser.WebService`. +5. Configure the CLI via environment variables (e.g. `STELLAOPS_BACKEND_URL`) and trigger + jobs with `dotnet run --project src/StellaOps.Cli -- db merge`. + +Detailed operator guidance is available in `docs/10_FEEDSER_CLI_QUICKSTART.md`. API and +command reference material lives in `docs/09_API_CLI_REFERENCE.md`. + +Pipeline note: deployment workflows should template `etc/feedser.yaml` during CI/CD, +injecting environment-specific Mongo credentials and telemetry endpoints. Upcoming +releases will add Microsoft OAuth (Entra ID) authentication support—track the quickstart +for integration steps once available. + diff --git a/SPRINTS.md b/SPRINTS.md new file mode 100644 index 00000000..80513e53 --- /dev/null +++ b/SPRINTS.md @@ -0,0 +1,95 @@ +| Sprint | Theme | Tasks File Path | Status | Type of Specialist | Task ID | Task Description | +| --- | --- | --- | --- | --- | --- | --- | +| Sprint 1 | Stabilize In-Progress Foundations | src/StellaOps.Feedser.Models/TASKS.md | — | Team Models & Merge Leads | FEEDMODELS-SCHEMA-01-001 | SemVer primitive range-style metadata
Instructions to work:
DONE Read ./AGENTS.md and src/StellaOps.Feedser.Models/AGENTS.md. This task lays the groundwork—complete the SemVer helper updates before teammates pick up FEEDMODELS-SCHEMA-01-002/003 and FEEDMODELS-SCHEMA-02-900. Use ./src/FASTER_MODELING_AND_NORMALIZATION.md for the target rule structure. | +| Sprint 1 | Stabilize In-Progress Foundations | src/StellaOps.Feedser.Models/TASKS.md | DONE (2025-10-11) | Team Models & Merge Leads | FEEDMODELS-SCHEMA-01-002 | Provenance decision rationale field
Instructions to work:
AdvisoryProvenance now carries `decisionReason` and docs/tests were updated. Connectors and merge tasks should populate the field when applying precedence/freshness/tie-breaker logic; see src/StellaOps.Feedser.Models/PROVENANCE_GUIDELINES.md for usage guidance. | +| Sprint 1 | Stabilize In-Progress Foundations | src/StellaOps.Feedser.Models/TASKS.md | DONE (2025-10-11) | Team Models & Merge Leads | FEEDMODELS-SCHEMA-01-003 | Normalized version rules collection
Instructions to work:
`AffectedPackage.NormalizedVersions` and supporting comparer/docs/tests shipped. Connector owners must emit rule arrays per ./src/FASTER_MODELING_AND_NORMALIZATION.md and report progress via FEEDMERGE-COORD-02-900 so merge/storage backfills can proceed. | +| Sprint 1 | Stabilize In-Progress Foundations | src/StellaOps.Feedser.Models/TASKS.md | — | Team Models & Merge Leads | FEEDMODELS-SCHEMA-02-900 | Range primitives for SemVer/EVR/NEVRA metadata
Instructions to work:
DONE Read ./AGENTS.md and src/StellaOps.Feedser.Models/AGENTS.md before resuming this stalled effort. Confirm helpers align with the new `NormalizedVersions` representation so connectors finishing in Sprint 2 can emit consistent metadata. | +| Sprint 1 | Stabilize In-Progress Foundations | src/StellaOps.Feedser.Normalization/TASKS.md | DONE (2025-10-11) | Team Normalization & Storage Backbone | FEEDNORM-NORM-02-001 | SemVer normalized rule emitter
Shared `SemVerRangeRuleBuilder` now outputs primitives + normalized rules per `FASTER_MODELING_AND_NORMALIZATION.md`; CVE/GHSA connectors consuming the API have verified fixtures. | +| Sprint 1 | Stabilize In-Progress Foundations | src/StellaOps.Feedser.Storage.Mongo/TASKS.md | DONE (2025-10-11) | Team Normalization & Storage Backbone | FEEDSTORAGE-DATA-02-001 | Normalized range dual-write + backfill
AdvisoryStore dual-writes flattened `normalizedVersions` when `feedser.storage.enableSemVerStyle` is set; migration `20251011-semver-style-backfill` updates historical records and docs outline the rollout. | +| Sprint 1 | Stabilize In-Progress Foundations | src/StellaOps.Feedser.Storage.Mongo/TASKS.md | DONE (2025-10-11) | Team Normalization & Storage Backbone | FEEDSTORAGE-DATA-02-002 | Provenance decision reason persistence
Storage now persists `provenance.decisionReason` for advisories and merge events; tests cover round-trips. | +| Sprint 1 | Stabilize In-Progress Foundations | src/StellaOps.Feedser.Storage.Mongo/TASKS.md | DONE (2025-10-11) | Team Normalization & Storage Backbone | FEEDSTORAGE-DATA-02-003 | Normalized versions indexing
Bootstrapper seeds compound/sparse indexes for flattened normalized rules and `docs/dev/mongo_indices.md` documents query guidance. | +| Sprint 1 | Stabilize In-Progress Foundations | src/StellaOps.Feedser.Storage.Mongo/TASKS.md | DONE (2025-10-11) | Team Normalization & Storage Backbone | FEEDSTORAGE-TESTS-02-004 | Restore AdvisoryStore build after normalized versions refactor
Updated constructors/tests keep storage suites passing with the new feature flag defaults. | +| Sprint 1 | Stabilize In-Progress Foundations | src/StellaOps.Feedser.WebService/TASKS.md | DONE (2025-10-11) | Team WebService & Authority | FEEDWEB-ENGINE-01-002 | Plumb Authority client resilience options
WebService now binds `authority.resilience.*` into `AddStellaOpsAuthClient`; integration tests verify retry/offline tolerance wiring. | +| Sprint 1 | Stabilize In-Progress Foundations | src/StellaOps.Feedser.WebService/TASKS.md | DONE (2025-10-11) | Team WebService & Authority | FEEDWEB-DOCS-01-003 | Author ops guidance for resilience tuning
Quickstart/install docs ship `authority.resilience.*` defaults, online vs air-gapped tuning, and monitoring references aligned with WebService coverage. | +| Sprint 1 | Stabilize In-Progress Foundations | src/StellaOps.Feedser.WebService/TASKS.md | DONE (2025-10-11) | Team WebService & Authority | FEEDWEB-DOCS-01-004 | Document authority bypass logging patterns
Guides now include the structured audit log fields (`route/status/subject/clientId/scopes/bypass/remote`) and SIEM alert guidance. | +| Sprint 1 | Stabilize In-Progress Foundations | src/StellaOps.Feedser.WebService/TASKS.md | DONE (2025-10-11) | Team WebService & Authority | FEEDWEB-DOCS-01-005 | Update Feedser operator guide for enforcement cutoff
Operator guide details the 2025-12-31 cutoff checklist and env var rollout to disable `allowAnonymousFallback`. | +| Sprint 1 | Stabilize In-Progress Foundations | src/StellaOps.Authority/TASKS.md | DONE (2025-10-11) | Team WebService & Authority | SEC3.HOST | Rate limiter policy binding
Authority host now applies configuration-driven fixed windows to `/token`, `/authorize`, and `/internal/*`; integration tests assert 429 + `Retry-After` headers; docs/config samples refreshed for Docs guild diagrams. | +| Sprint 1 | Stabilize In-Progress Foundations | src/StellaOps.Authority/TASKS.md | DONE (2025-10-11) | Team WebService & Authority | SEC3.BUILD | Authority rate-limiter follow-through
`Security.RateLimiting` now fronts token/authorize/internal limiters; Authority + Configuration matrices (`dotnet test src/StellaOps.Authority/StellaOps.Authority.sln`, `dotnet test src/StellaOps.Configuration.Tests/StellaOps.Configuration.Tests.csproj`) passed on 2025-10-11; awaiting #authority-core broadcast. | +| Sprint 1 | Stabilize In-Progress Foundations | src/StellaOps.Authority/StellaOps.Authority.Plugin.Standard/TASKS.md | DONE (2025-10-11) | Team WebService & Authority | PLG6.DOC | Plugin developer guide polish
Section 9 now documents rate limiter metadata, config keys, and lockout interplay; YAML samples updated alongside Authority config templates. | +| Sprint 1 | Stabilize In-Progress Foundations | src/StellaOps.Feedser.Source.CertCc/TASKS.md | DONE (2025-10-11) | Team Connector Resumption – CERT/RedHat | FEEDCONN-CERTCC-02-001 | Fetch pipeline & state tracking
Summary planner now drives monthly/yearly VINCE fetches, persists pending summaries/notes, and hydrates VINCE detail queue with telemetry.
Team instructions: Read ./AGENTS.md and src/StellaOps.Feedser.Source.CertCc/AGENTS.md. Coordinate daily with Models/Merge leads so new normalizedVersions output and provenance tags stay aligned with ./src/FASTER_MODELING_AND_NORMALIZATION.md. | +| Sprint 1 | Stabilize In-Progress Foundations | src/StellaOps.Feedser.Source.CertCc/TASKS.md | DONE (2025-10-11) | Team Connector Resumption – CERT/RedHat | FEEDCONN-CERTCC-02-002 | VINCE note detail fetcher
Summary planner queues VINCE note detail endpoints, persists raw JSON with SHA/ETag metadata, and records retry/backoff metrics. | +| Sprint 1 | Stabilize In-Progress Foundations | src/StellaOps.Feedser.Source.CertCc/TASKS.md | DONE (2025-10-11) | Team Connector Resumption – CERT/RedHat | FEEDCONN-CERTCC-02-003 | DTO & parser implementation
Added VINCE DTO aggregate, Markdown→text sanitizer, vendor/status/vulnerability parsers, and parser regression fixture. | +| Sprint 1 | Stabilize In-Progress Foundations | src/StellaOps.Feedser.Source.CertCc/TASKS.md | DONE (2025-10-11) | Team Connector Resumption – CERT/RedHat | FEEDCONN-CERTCC-02-004 | Canonical mapping & range primitives
VINCE DTO aggregate flows through `CertCcMapper`, emitting vendor range primitives + normalized version rules that persist via `_advisoryStore`. | +| Sprint 1 | Stabilize In-Progress Foundations | src/StellaOps.Feedser.Source.CertCc/TASKS.md | DOING (2025-10-11) | Team Connector Resumption – CERT/RedHat | FEEDCONN-CERTCC-02-005 | Deterministic fixtures/tests
Fetch→parse→map integration suite in place; snapshot harness reactivation pending fixture regeneration & documentation of the `UPDATE_CERTCC_FIXTURES` flow. | +| Sprint 1 | Stabilize In-Progress Foundations | src/StellaOps.Feedser.Source.CertCc/TASKS.md | DONE (2025-10-11) | Team Connector Resumption – CERT/RedHat | FEEDCONN-CERTCC-02-006 | Telemetry & documentation
WebService now exports the `StellaOps.Feedser.Source.CertCc` meter and fetch integration tests assert `certcc.*` counters; README updated with observability guidance. | +| Sprint 1 | Stabilize In-Progress Foundations | src/StellaOps.Feedser.Source.CertCc/TASKS.md | DOING (2025-10-11) | Team Connector Resumption – CERT/RedHat | FEEDCONN-CERTCC-02-007 | Connector test harness remediation
Canned-response harness restored; snapshot fixtures still being realigned and regeneration steps documented. | +| Sprint 1 | Stabilize In-Progress Foundations | src/StellaOps.Feedser.Source.CertCc/TASKS.md | BLOCKED (2025-10-11) | Team Connector Resumption – CERT/RedHat | FEEDCONN-CERTCC-02-008 | Snapshot coverage handoff
Upstream repo version lacks SemVer primitives + provenance decision reason fields, so snapshot regeneration fails; resume once Models/Storage sprint lands those changes. | +| Sprint 1 | Stabilize In-Progress Foundations | src/StellaOps.Feedser.Source.CertCc/TASKS.md | TODO | Team Connector Resumption – CERT/RedHat | FEEDCONN-CERTCC-02-012 | Schema sync & snapshot regen follow-up
Re-run fixture updates and deliver Merge handoff after SemVer style + decision reason updates merge into main. | +| Sprint 1 | Stabilize In-Progress Foundations | src/StellaOps.Feedser.Source.CertCc/TASKS.md | DONE (2025-10-11) | Team Connector Resumption – CERT/RedHat | FEEDCONN-CERTCC-02-009 | Detail/map reintegration plan
Staged reintegration plan published in `src/StellaOps.Feedser.Source.CertCc/FEEDCONN-CERTCC-02-009_PLAN.md`; coordinates enablement with FEEDCONN-CERTCC-02-004. | +| Sprint 1 | Stabilize In-Progress Foundations | src/StellaOps.Feedser.Source.CertCc/TASKS.md | TODO (2025-10-11) | Team Connector Resumption – CERT/RedHat | FEEDCONN-CERTCC-02-010 | Partial-detail graceful degradation
Ensure missing VINCE endpoints downgrade gracefully without fatal errors; add coverage for partial note ingestion paths. | +| Sprint 1 | Stabilize In-Progress Foundations | src/StellaOps.Feedser.Source.Distro.RedHat/TASKS.md | DONE (2025-10-11) | Team Connector Resumption – CERT/RedHat | FEEDCONN-REDHAT-02-001 | Fixture validation sweep
Instructions to work:
Fixtures regenerated post-model-helper rollout; provenance ordering and normalizedVersions scaffolding verified via tests. Conflict resolver deltas logged in src/StellaOps.Feedser.Source.Distro.RedHat/CONFLICT_RESOLVER_NOTES.md for Sprint 3 consumers. | +| Sprint 1 | Stabilize In-Progress Foundations | src/StellaOps.Feedser.Source.Vndr.Apple/TASKS.md | — | Team Vendor Apple Specialists | FEEDCONN-APPLE-02-001 | Canonical mapping & range primitives | +| Sprint 1 | Stabilize In-Progress Foundations | src/StellaOps.Feedser.Source.Vndr.Apple/TASKS.md | DONE (2025-10-11) | Team Vendor Apple Specialists | FEEDCONN-APPLE-02-002 | Deterministic fixtures/tests
Sanitized live fixtures + regression snapshots wired into tests; normalized rule coverage asserted. | +| Sprint 1 | Stabilize In-Progress Foundations | src/StellaOps.Feedser.Source.Vndr.Apple/TASKS.md | DONE (2025-10-11) | Team Vendor Apple Specialists | FEEDCONN-APPLE-02-003 | Telemetry & documentation
Apple meter metrics wired into Feedser WebService OpenTelemetry configuration; README and fixtures document normalizedVersions coverage. | +| Sprint 1 | Stabilize In-Progress Foundations | src/StellaOps.Feedser.Source.Vndr.Apple/TASKS.md | DONE (2025-10-11) | Team Vendor Apple Specialists | FEEDCONN-APPLE-02-004 | Live HTML regression sweep
Live support.apple.com sweep captured iOS/macOS/RSR cases with deterministic sanitizers. | +| Sprint 1 | Stabilize In-Progress Foundations | src/StellaOps.Feedser.Source.Vndr.Apple/TASKS.md | DONE (2025-10-11) | Team Vendor Apple Specialists | FEEDCONN-APPLE-02-005 | Fixture regeneration tooling
`UPDATE_APPLE_FIXTURES=1` flow fetches & rewrites fixtures; README documents usage.
Instructions to work:
DONE Read ./AGENTS.md and src/StellaOps.Feedser.Source.Vndr.Apple/AGENTS.md. Resume stalled tasks, ensuring normalizedVersions output and fixtures align with ./src/FASTER_MODELING_AND_NORMALIZATION.md before handing data to the conflict sprint. | +| Sprint 1 | Stabilize In-Progress Foundations | src/StellaOps.Feedser.Source.Ghsa/TASKS.md | — | Team Connector Normalized Versions Rollout | FEEDCONN-GHSA-02-001 | GHSA normalized versions & provenance
Team instructions: Read ./AGENTS.md and each module's AGENTS file. Adopt the `NormalizedVersions` array emitted by the models sprint, wiring provenance `decisionReason` where merge overrides occur. Follow ./src/FASTER_MODELING_AND_NORMALIZATION.md; report via src/StellaOps.Feedser.Merge/TASKS.md (FEEDMERGE-COORD-02-900). Progress 2025-10-11: GHSA/OSV emit normalized arrays with refreshed fixtures; CVE mapper now surfaces SemVer normalized ranges; NVD/KEV adoption pending; outstanding follow-ups include FEEDSTORAGE-DATA-02-001, FEEDMERGE-ENGINE-02-002, and rolling `tools/FixtureUpdater` updates across connectors. | +| Sprint 1 | Stabilize In-Progress Foundations | src/StellaOps.Feedser.Source.Osv/TASKS.md | — | Team Connector Normalized Versions Rollout | FEEDCONN-OSV-02-003 | OSV normalized versions & freshness | +| Sprint 1 | Stabilize In-Progress Foundations | src/StellaOps.Feedser.Source.Nvd/TASKS.md | — | Team Connector Normalized Versions Rollout | FEEDCONN-NVD-02-002 | NVD normalized versions & timestamps | +| Sprint 1 | Stabilize In-Progress Foundations | src/StellaOps.Feedser.Source.Cve/TASKS.md | — | Team Connector Normalized Versions Rollout | FEEDCONN-CVE-02-003 | CVE normalized versions uplift | +| Sprint 1 | Stabilize In-Progress Foundations | src/StellaOps.Feedser.Source.Kev/TASKS.md | — | Team Connector Normalized Versions Rollout | FEEDCONN-KEV-02-003 | KEV normalized versions propagation | +| Sprint 1 | Stabilize In-Progress Foundations | src/StellaOps.Feedser.Source.Osv/TASKS.md | — | Team Connector Normalized Versions Rollout | FEEDCONN-OSV-04-003 | OSV parity fixture refresh | +| Sprint 1 | Stabilize In-Progress Foundations | src/StellaOps.Feedser.WebService/TASKS.md | DONE (2025-10-10) | Team WebService & Authority | FEEDWEB-DOCS-01-001 | Document authority toggle & scope requirements
Quickstart carries toggle/scope guidance pending docs guild review (no change this sprint). | +| Sprint 1 | Stabilize In-Progress Foundations | src/StellaOps.Feedser.WebService/TASKS.md | DONE (2025-10-11) | Team WebService & Authority | FEEDWEB-ENGINE-01-002 | Plumb Authority client resilience options
WebService binds `authority.resilience.*` into `AddStellaOpsAuthClient`; integration tests validate retry/offline wiring. | +| Sprint 1 | Stabilize In-Progress Foundations | src/StellaOps.Feedser.WebService/TASKS.md | DONE (2025-10-11) | Team WebService & Authority | FEEDWEB-DOCS-01-003 | Author ops guidance for resilience tuning
Operator docs cover resilience defaults, environment-specific tuning, and monitoring cues. | +| Sprint 1 | Stabilize In-Progress Foundations | src/StellaOps.Feedser.WebService/TASKS.md | DONE (2025-10-11) | Team WebService & Authority | FEEDWEB-DOCS-01-004 | Document authority bypass logging patterns
Audit logging examples now document bypass fields and recommended alerts. | +| Sprint 1 | Stabilize In-Progress Foundations | src/StellaOps.Feedser.WebService/TASKS.md | DONE (2025-10-11) | Team WebService & Authority | FEEDWEB-DOCS-01-005 | Update Feedser operator guide for enforcement cutoff
Enforcement checklist and env overrides documented ahead of the 2025-12-31 cutoff. | +| Sprint 1 | Stabilize In-Progress Foundations | src/StellaOps.Feedser.WebService/TASKS.md | DONE (2025-10-11) | Team WebService & Authority | FEEDWEB-OPS-01-006 | Rename plugin drop directory to namespaced path
Build outputs, tests, and docs now target `StellaOps.Feedser.PluginBinaries`/`StellaOps.Authority.PluginBinaries`. | +| Sprint 1 | Stabilize In-Progress Foundations | src/StellaOps.Feedser.WebService/TASKS.md | DONE (2025-10-11) | Team WebService & Authority | FEEDWEB-OPS-01-007 | Authority resilience adoption
Deployment docs and CLI notes explain the LIB5 resilience knobs for rollout.
Instructions to work:
DONE Read ./AGENTS.md and src/StellaOps.Feedser.WebService/AGENTS.md. These items were mid-flight; resume implementation ensuring docs/operators receive timely updates. | +| Sprint 1 | Stabilize In-Progress Foundations | src/StellaOps.Authority/TASKS.md | DONE (2025-10-11) | Team Authority Platform & Security Guild | AUTHCORE-ENGINE-01-001 | CORE8.RL — Rate limiter plumbing validated; integration tests green and docs handoff recorded for middleware ordering + Retry-After headers (see `docs/dev/authority-rate-limit-tuning-outline.md` for continuing guidance). | +| Sprint 1 | Stabilize In-Progress Foundations | src/StellaOps.Cryptography/TASKS.md | DONE (2025-10-11) | Team Authority Platform & Security Guild | AUTHCRYPTO-ENGINE-01-001 | SEC3.A — Shared metadata resolver confirmed via host test run; SEC3.B now unblocked for tuning guidance (outline captured in `docs/dev/authority-rate-limit-tuning-outline.md`). | +| Sprint 1 | Stabilize In-Progress Foundations | src/StellaOps.Authority/StellaOps.Authority.Plugin.Standard/TASKS.md | DOING (2025-10-11) | Team Authority Platform & Security Guild | AUTHPLUG-DOCS-01-001 | PLG6.DOC — Docs guild resuming diagram/copy updates using the captured limiter context + configuration notes (reference `docs/dev/authority-rate-limit-tuning-outline.md` for tuning matrix + observability copy).
Instructions to work:
Read ./AGENTS.md plus module-specific AGENTS. Restart the blocked rate-limiter workstream (Authority host + cryptography) so the plugin docs team can finish diagrams. Coordinate daily; use ./src/DEDUP_CONFLICTS_RESOLUTION_ALGO.md where rate limiting interacts with conflict policy. | +| Sprint 2 | Connector & Data Implementation Wave | src/StellaOps.Feedser.Normalization/TASKS.md | — | Team Normalization & Storage Backbone | FEEDNORM-NORM-02-001 | SemVer normalized rule emitter
Instructions to work:
Read ./AGENTS.md and module AGENTS. Use ./src/FASTER_MODELING_AND_NORMALIZATION.md to build the shared rule generator; sync daily with storage and connector owners. | +| Sprint 2 | Connector & Data Implementation Wave | src/StellaOps.Feedser.Storage.Mongo/TASKS.md | — | Team Normalization & Storage Backbone | FEEDSTORAGE-DATA-02-001 | Normalized range dual-write + backfill | +| Sprint 2 | Connector & Data Implementation Wave | src/StellaOps.Feedser.Storage.Mongo/TASKS.md | — | Team Normalization & Storage Backbone | FEEDSTORAGE-DATA-02-002 | Provenance decision reason persistence | +| Sprint 2 | Connector & Data Implementation Wave | src/StellaOps.Feedser.Storage.Mongo/TASKS.md | — | Team Normalization & Storage Backbone | FEEDSTORAGE-DATA-02-003 | Normalized versions indexing
Instructions to work:
Read ./AGENTS.md and storage AGENTS. Implement dual-write/backfill and index creation using the shapes from ./src/FASTER_MODELING_AND_NORMALIZATION.md; coordinate with connectors entering the sprint. | +| Sprint 2 | Connector & Data Implementation Wave | src/StellaOps.Feedser.Merge/TASKS.md | DONE (2025-10-11) | Team Normalization & Storage Backbone | FEEDMERGE-ENGINE-02-002 | Normalized versions union & dedupe
Affected package resolver unions/dedupes normalized rules, stamps merge provenance with `decisionReason`, and tests cover the rollout. | +| Sprint 2 | Connector & Data Implementation Wave | src/StellaOps.Feedser.Source.Ghsa/TASKS.md | DONE (2025-10-11) | Team Connector Expansion – GHSA/NVD/OSV | FEEDCONN-GHSA-02-001 | GHSA normalized versions & provenance | +| Sprint 2 | Connector & Data Implementation Wave | src/StellaOps.Feedser.Source.Ghsa/TASKS.md | DONE (2025-10-11) | Team Connector Expansion – GHSA/NVD/OSV | FEEDCONN-GHSA-02-004 | GHSA credits & ecosystem severity mapping | +| Sprint 2 | Connector & Data Implementation Wave | src/StellaOps.Feedser.Source.Ghsa/TASKS.md | TODO | Team Connector Expansion – GHSA/NVD/OSV | FEEDCONN-GHSA-02-005 | GitHub quota monitoring & retries | +| Sprint 2 | Connector & Data Implementation Wave | src/StellaOps.Feedser.Source.Ghsa/TASKS.md | TODO | Team Connector Expansion – GHSA/NVD/OSV | FEEDCONN-GHSA-02-006 | Production credential & scheduler rollout | +| Sprint 2 | Connector & Data Implementation Wave | src/StellaOps.Feedser.Source.Ghsa/TASKS.md | TODO | Team Connector Expansion – GHSA/NVD/OSV | FEEDCONN-GHSA-02-007 | Credit parity regression fixtures | +| Sprint 2 | Connector & Data Implementation Wave | src/StellaOps.Feedser.Source.Nvd/TASKS.md | DONE (2025-10-11) | Team Connector Expansion – GHSA/NVD/OSV | FEEDCONN-NVD-02-002 | NVD normalized versions & timestamps | +| Sprint 2 | Connector & Data Implementation Wave | src/StellaOps.Feedser.Source.Nvd/TASKS.md | DONE (2025-10-11) | Team Connector Expansion – GHSA/NVD/OSV | FEEDCONN-NVD-02-004 | NVD CVSS & CWE precedence payloads | +| Sprint 2 | Connector & Data Implementation Wave | src/StellaOps.Feedser.Source.Nvd/TASKS.md | TODO | Team Connector Expansion – GHSA/NVD/OSV | FEEDCONN-NVD-02-005 | NVD merge/export parity regression | +| Sprint 2 | Connector & Data Implementation Wave | src/StellaOps.Feedser.Source.Osv/TASKS.md | DONE (2025-10-11) | Team Connector Expansion – GHSA/NVD/OSV | FEEDCONN-OSV-02-003 | OSV normalized versions & freshness | +| Sprint 2 | Connector & Data Implementation Wave | src/StellaOps.Feedser.Source.Osv/TASKS.md | DONE (2025-10-11) | Team Connector Expansion – GHSA/NVD/OSV | FEEDCONN-OSV-02-004 | OSV references & credits alignment | +| Sprint 2 | Connector & Data Implementation Wave | src/StellaOps.Feedser.Source.Osv/TASKS.md | TODO | Team Connector Expansion – GHSA/NVD/OSV | FEEDCONN-OSV-02-005 | Fixture updater workflow
Instructions to work:
Read ./AGENTS.md and respective module AGENTS. Implement builder integration, provenance, and supporting docs using ./src/FASTER_MODELING_AND_NORMALIZATION.md and ensure outputs satisfy the precedence matrix in ./src/DEDUP_CONFLICTS_RESOLUTION_ALGO.md. | +| Sprint 2 | Connector & Data Implementation Wave | src/StellaOps.Feedser.Source.Acsc/TASKS.md | Research DOING | Team Connector Expansion – Regional & Vendor Feeds | FEEDCONN-ACSC-02-001 … 02-008 | RSS endpoints catalogued 2025-10-11, HTTP/2 error + pagination validation and client compatibility task opened. | +| Sprint 2 | Connector & Data Implementation Wave | src/StellaOps.Feedser.Source.Cccs/TASKS.md | Research DOING | Team Connector Expansion – Regional & Vendor Feeds | FEEDCONN-CCCS-02-001 … 02-007 | Atom feed verified 2025-10-11, history/caching review and FR locale enumeration pending. | +| Sprint 2 | Connector & Data Implementation Wave | src/StellaOps.Feedser.Source.CertBund/TASKS.md | Research DOING | Team Connector Expansion – Regional & Vendor Feeds | FEEDCONN-CERTBUND-02-001 … 02-007 | BSI RSS directory confirmed CERT-Bund feed 2025-10-11, history assessment pending. | +| Sprint 2 | Connector & Data Implementation Wave | src/StellaOps.Feedser.Source.Kisa/TASKS.md | Research DOING | Team Connector Expansion – Regional & Vendor Feeds | FEEDCONN-KISA-02-001 … 02-007 | KNVD RSS endpoint identified 2025-10-11, access headers/session strategy outstanding. | +| Sprint 2 | Connector & Data Implementation Wave | src/StellaOps.Feedser.Source.Ru.Bdu/TASKS.md | Research DOING | Team Connector Expansion – Regional & Vendor Feeds | FEEDCONN-RUBDU-02-001 … 02-008 | BDU RSS/Atom catalogue reviewed 2025-10-11, trust-store acquisition blocked by gosuslugi placeholder page. | +| Sprint 2 | Connector & Data Implementation Wave | src/StellaOps.Feedser.Source.Ru.Nkcki/TASKS.md | Research DOING | Team Connector Expansion – Regional & Vendor Feeds | FEEDCONN-NKCKI-02-001 … 02-008 | cert.gov.ru paginated RSS landing checked 2025-10-11, access enablement plan pending. | +| Sprint 2 | Connector & Data Implementation Wave | src/StellaOps.Feedser.Source.Ics.Cisa/TASKS.md | Research DOING | Team Connector Expansion – Regional & Vendor Feeds | FEEDCONN-ICSCISA-02-001 … 02-008 | new ICS RSS endpoint logged 2025-10-11 but Akamai blocks direct pulls, fallback strategy task opened. | +| Sprint 2 | Connector & Data Implementation Wave | src/StellaOps.Feedser.Source.Vndr.Cisco/TASKS.md | Research DOING | Team Connector Expansion – Regional & Vendor Feeds | FEEDCONN-CISCO-02-001 … 02-007 | openVuln API + RSS reviewed 2025-10-11, auth/pagination memo pending. | +| Sprint 2 | Connector & Data Implementation Wave | src/StellaOps.Feedser.Source.Vndr.Msrc/TASKS.md | Research DOING | Team Connector Expansion – Regional & Vendor Feeds | FEEDCONN-MSRC-02-001 … 02-007 | MSRC API docs reviewed 2025-10-11, auth/throttling comparison memo pending.
Instructions to work:
Read ./AGENTS.md plus each module's AGENTS file. Parallelize research, ingestion, mapping, fixtures, and docs using the normalized rule shape from ./src/FASTER_MODELING_AND_NORMALIZATION.md. Coordinate daily with the merge coordination task from Sprint 1. | +| Sprint 2 | Connector & Data Implementation Wave | src/StellaOps.Feedser.Source.Cve/TASKS.md | — | Team Connector Support & Monitoring | FEEDCONN-CVE-02-001 … 02-002 | Instructions to work:
Read ./AGENTS.md and module AGENTS. Deliver operator docs and monitoring instrumentation required for broader feed rollout. | +| Sprint 2 | Connector & Data Implementation Wave | src/StellaOps.Feedser.Source.Kev/TASKS.md | — | Team Connector Support & Monitoring | FEEDCONN-KEV-02-001 … 02-002 | Instructions to work:
Read ./AGENTS.md and module AGENTS. Deliver operator docs and monitoring instrumentation required for broader feed rollout. | +| Sprint 2 | Connector & Data Implementation Wave | docs/TASKS.md | DONE (2025-10-11) | Team Docs & Knowledge Base | FEEDDOCS-DOCS-01-001 | Canonical schema docs refresh
Updated canonical schema + provenance guides with SemVer style, normalized version rules, decision reason change log, and migration notes. | +| Sprint 2 | Connector & Data Implementation Wave | docs/TASKS.md | DONE (2025-10-11) | Team Docs & Knowledge Base | FEEDDOCS-DOCS-02-001 | Feedser-SemVer Playbook
Published merge playbook covering mapper patterns, dedupe flow, indexes, and rollout checklist. | +| Sprint 2 | Connector & Data Implementation Wave | docs/TASKS.md | DONE (2025-10-11) | Team Docs & Knowledge Base | FEEDDOCS-DOCS-02-002 | Normalized versions query guide
Delivered Mongo index/query addendum with `$unwind` recipes, dedupe checks, and operational checklist.
Instructions to work:
DONE Read ./AGENTS.md and docs/AGENTS.md. Document every schema/index/query change produced in Sprint 1-2 leveraging ./src/FASTER_MODELING_AND_NORMALIZATION.md. | +| Sprint 3 | Conflict Resolution Integration & Communications | src/StellaOps.Feedser.Core/TASKS.md | DONE (2025-10-11) | Team Core Engine & Storage Analytics | FEEDCORE-ENGINE-03-001 | Canonical merger implementation
`CanonicalMerger` ships with freshness/tie-breaker logic, provenance, and unit coverage feeding Merge. | +| Sprint 3 | Conflict Resolution Integration & Communications | src/StellaOps.Feedser.Core/TASKS.md | DONE (2025-10-11) | Team Core Engine & Storage Analytics | FEEDCORE-ENGINE-03-002 | Field precedence and tie-breaker map
Field precedence tables and tie-breaker metrics wired into the canonical merge flow; docs/tests updated.
Instructions to work:
Read ./AGENTS.md and core AGENTS. Implement the conflict resolver exactly as specified in ./src/DEDUP_CONFLICTS_RESOLUTION_ALGO.md, coordinating with Merge and Storage teammates. | +| Sprint 3 | Conflict Resolution Integration & Communications | src/StellaOps.Feedser.Storage.Mongo/TASKS.md | DONE (2025-10-11) | Team Core Engine & Storage Analytics | FEEDSTORAGE-DATA-03-001 | Merge event provenance audit prep
Merge events now persist `fieldDecisions` and analytics-ready provenance snapshots. | +| Sprint 3 | Conflict Resolution Integration & Communications | src/StellaOps.Feedser.Storage.Mongo/TASKS.md | DONE (2025-10-11) | Team Core Engine & Storage Analytics | FEEDSTORAGE-DATA-02-001 | Normalized range dual-write + backfill
Dual-write/backfill flag delivered; migration + options validated in tests. | +| Sprint 3 | Conflict Resolution Integration & Communications | src/StellaOps.Feedser.Storage.Mongo/TASKS.md | DONE (2025-10-11) | Team Core Engine & Storage Analytics | FEEDSTORAGE-TESTS-02-004 | Restore AdvisoryStore build after normalized versions refactor
Storage tests adjusted for normalized versions/decision reasons.
Instructions to work:
Read ./AGENTS.md and storage AGENTS. Extend merge events with decision reasons and analytics views to support the conflict rules, and deliver the dual-write/backfill for `NormalizedVersions` + `decisionReason` so connectors can roll out safely. | +| Sprint 3 | Conflict Resolution Integration & Communications | src/StellaOps.Feedser.Merge/TASKS.md | DONE (2025-10-11) | Team Merge & QA Enforcement | FEEDMERGE-ENGINE-04-001 | GHSA/NVD/OSV conflict rules
Merge pipeline consumes `CanonicalMerger` output prior to precedence merge. | +| Sprint 3 | Conflict Resolution Integration & Communications | src/StellaOps.Feedser.Merge/TASKS.md | DONE (2025-10-11) | Team Merge & QA Enforcement | FEEDMERGE-ENGINE-04-002 | Override metrics instrumentation
Merge events capture per-field decisions; counters/logs align with conflict rules. | +| Sprint 3 | Conflict Resolution Integration & Communications | src/StellaOps.Feedser.Merge/TASKS.md | DONE (2025-10-11) | Team Merge & QA Enforcement | FEEDMERGE-ENGINE-04-003 | Reference & credit union pipeline
Canonical merge preserves unions with updated tests. | +| Sprint 3 | Conflict Resolution Integration & Communications | src/StellaOps.Feedser.Merge/TASKS.md | DONE (2025-10-11) | Team Merge & QA Enforcement | FEEDMERGE-QA-04-001 | End-to-end conflict regression suite
Added regression tests (`AdvisoryMergeServiceTests`) covering canonical + precedence flow.
Instructions to work:
Read ./AGENTS.md and merge AGENTS. Integrate the canonical merger, instrument metrics, and deliver comprehensive regression tests following ./src/DEDUP_CONFLICTS_RESOLUTION_ALGO.md. | +| Sprint 3 | Conflict Resolution Integration & Communications | src/StellaOps.Feedser.Source.Ghsa/TASKS.md | — | Team Connector Regression Fixtures | FEEDCONN-GHSA-04-002 | GHSA conflict regression fixtures | +| Sprint 3 | Conflict Resolution Integration & Communications | src/StellaOps.Feedser.Source.Nvd/TASKS.md | — | Team Connector Regression Fixtures | FEEDCONN-NVD-04-002 | NVD conflict regression fixtures | +| Sprint 3 | Conflict Resolution Integration & Communications | src/StellaOps.Feedser.Source.Osv/TASKS.md | — | Team Connector Regression Fixtures | FEEDCONN-OSV-04-002 | OSV conflict regression fixtures
Instructions to work:
Read ./AGENTS.md and module AGENTS. Produce fixture triples supporting the precedence/tie-breaker paths defined in ./src/DEDUP_CONFLICTS_RESOLUTION_ALGO.md and hand them to Merge QA. | +| Sprint 3 | Conflict Resolution Integration & Communications | docs/TASKS.md | DONE (2025-10-11) | Team Documentation Guild – Conflict Guidance | FEEDDOCS-DOCS-05-001 | Feedser Conflict Rules
Runbook published at `docs/ops/feedser-conflict-resolution.md`; metrics/log guidance aligned with Sprint 3 merge counters. | +| Sprint 3 | Conflict Resolution Integration & Communications | docs/TASKS.md | TODO | Team Documentation Guild – Conflict Guidance | FEEDDOCS-DOCS-05-002 | Conflict runbook ops rollout
Instructions to work:
Read ./AGENTS.md and docs/AGENTS.md. Socialise the conflict runbook with Feedser Ops, tune alert thresholds, and record change-log linkage once sign-off is captured. Use ./src/DEDUP_CONFLICTS_RESOLUTION_ALGO.md for ongoing rule references. | diff --git a/TODOS.md b/TODOS.md new file mode 100644 index 00000000..86403496 --- /dev/null +++ b/TODOS.md @@ -0,0 +1,12 @@ +# Current Focus – FEEDCONN-CERTCC +| Task | Status | Notes | +|---|---|---| +|FEEDCONN-CERTCC-02-005 Deterministic fixtures/tests|DONE (2025-10-11)|Snapshot regression for summary/detail fetch landed; fixtures regenerate via `UPDATE_CERTCC_FIXTURES`.| +|FEEDCONN-CERTCC-02-008 Snapshot coverage handoff|DONE (2025-10-11)|`CertCcConnectorSnapshotTests` produce documents/state/request snapshots and document the refresh workflow.| +|FEEDCONN-CERTCC-02-007 Connector test harness remediation|TODO|Need to restore Source.CertCc harness (`AddSourceCommon`, canned responses) so parser regression can run in CI.| +|FEEDCONN-CERTCC-02-009 Detail/map reintegration plan|DONE (2025-10-11)|Plan published in `src/StellaOps.Feedser.Source.CertCc/FEEDCONN-CERTCC-02-009_PLAN.md`; outlines staged enablement + rollback.| + +# Connector Apple Status +| Task | Status | Notes | +|---|---|---| +|FEEDCONN-APPLE-02-003 Telemetry & documentation|DONE (2025-10-11)|Apple connector meter registered with WebService OpenTelemetry metrics; README and fixtures highlight normalizedVersions coverage for conflict sprint handoff.| diff --git a/certificates/russian_trusted_sub_ca.cer b/certificates/russian_trusted_sub_ca.cer new file mode 100644 index 00000000..bfb4e7dc --- /dev/null +++ b/certificates/russian_trusted_sub_ca.cer @@ -0,0 +1,10 @@ +Title: Госуслуги + +URL Source: https://www.gosuslugi.ru/tls/files/subca2022.cer + +Markdown Content: +### Госуслуги сейчас откроются + +Портал работает в прежнем режиме. + + Подождите пару секунд diff --git a/docs/01_WHAT_IS_IT.md b/docs/01_WHAT_IS_IT.md new file mode 100755 index 00000000..90e2bf86 --- /dev/null +++ b/docs/01_WHAT_IS_IT.md @@ -0,0 +1,77 @@ +# 1 · What Is - **Stella Ops**? + +Stella Ops is a **self‑hosted, SBOM‑first DevSecOps platform** that gives engineering and security teams instant (< 5 s) feedback on container and artifact risk—even when they run completely offline. +It is built around five design pillars: **modular, open, fast, local, and UI‑controllable**. + +--- + +## 1. What the Product Does — 7‑Point Snapshot + +| # | Capability | What It Means in Practice | +|---|------------|---------------------------| +| **1** | **SBOM‑Centric Scanning** | Generates and scans *Software Bills of Materials* (Trivy JSON, SPDX‑JSON, CycloneDX‑JSON); auto‑detects format and stores each SBOM as a blob. | +| **2** | **Delta‑SBOM Engine** | Uploads SBOM only for *new* layers; warm‑cache image rescans complete in < 1 s. | +| **3** | **Anonymous Internal Registry** | Ships a built‑in `StellaOps.Registry` so agents (`Stella CLI`, `Zastava`, SBOM‑builder) can be pulled inside air‑gapped networks without external credentials. | +| **4** | **Policy‑as‑Code** | Supports YAML rules today and OPA/Rego (`StellaOps.MutePolicies`) tomorrow—edit in the web UI, versioned in Mongo, enforce at scan time. | +| **5** | **Pluggable Modules** | Every scanner, exporter, or attestor is a hot‑load .NET plug‑in (e.g., `StellaOpsAttestor` for SLSA/Rekor in the roadmap). | +| **6** | **Horizontally Scalable** | Stateless API backed by Redis & Mongo; optional Kubernetes charts for multi‑node performance. | +| **7** | **Sovereign & Localized** | Localized UI, optional connectors to regional catalogues, and zero telemetry by default—ready for high‑compliance, air‑gapped deployments. | + +> **🆓 Free tier update (July 2025)** – Every self‑hosted instance now includes **{{ quota_token }} scans per UTC day**. +> A yellow banner appears once you cross **200 scans** (≈ 60 % of quota). +> Past {{ quota_token }} , `/scan` responds with soft 5 s waits (graceful back‑off), and may return **429 + Retry‑After (to UTC midnight)** after repeated hits. + +--- + +## 2. How It Works — End‑to‑End Flow (30 sec tour) + +1. **Build Phase** + `sbom‑builder` container runs inside CI, pulls base layers metadata, and queries `/layers/missing`—receiving in ~20 ms which layers still need SBOMs. + • New layers ➟ SBOM generated ➟ `*.sbom.` + `*.sbom.type` dropped next to image tarball. + +2. **Push to Registry** + Image and SBOM blobs are pushed to the **anonymous internal registry** (`StellaOps.Registry`). Cosign tags are attached if enabled. + +3. **Scan Phase** + `Stella CLI` agent pulls the SBOM blob, sends `/scan?sbomType=spdx-json` to backend. If flag is absent, backend auto‑detects. + • Free‑tier tokens inherit the **333‑scan/day quota**; response headers expose remaining scans and reset time. + +4. **Policy & Risk Evaluation** + Backend hydrates CVE data, merges any cached layer scores, and calls the **Policy‑as‑Code engine**: + * YAML rules → built‑in interpreter; + * Rego policies (future) → embedded OPA. + +5. **Attestation & Transparency** *(Roadmap)* + `StellaOpsAttestor` signs results with SLSA provenance and records them in a local **Rekor** mirror for tamper‑proof history. + +6. **Feedback Loop** + • CLI exits with non‑zero on policy block. + • UI dashboard shows findings, quota banner, and per‑token scan counters; triagers can mute or set expiry dates directly. + +--- + +## 3. Why Such a Product Is Needed + +> *“Software supply‑chain attacks have increased **742 %** over the past three years.”* – Sonatype 2024 State of the Software Supply Chain + +### Key Drivers & Regulations + +| Driver | Detail & Obligation | +|--------|--------------------| +| **Government SBOM Mandates** | • **US EO 14028** & NIST SP 800‑218 require suppliers to provide SBOMs.
• EU **Cyber Resilience Act (CRA)** will demand attestations of secure development by 2026. | +| **SLSA & SSDF Frameworks** | Industry pushes toward **SLSA v1.0** levels 2‑3 and NIST **SSDF 1.1** controls, emphasising provenance and policy enforcement. | +| **Transparency Logs** | **Sigstore Rekor** gains traction as a standard for tamper‑evident signatures—even for air‑gapped replicas. | +| **Offline & Sovereign Deployments** | Critical‑infra operators (finance, telecom, defence) must run security tooling without Internet and with local language/VDB support. | +| **Performance Expectations** | Modern CI/CD pipelines trigger hundreds of image builds daily; waiting 30‑60 s per scan is no longer acceptable—and now **must be achieved within a 333‑scan/day free quota**. | + +### Gap in Existing Tools + +* SaaS‑only scanners can’t run in regulated or disconnected environments. +* Monolithic open‑source scanners are hard‑wired to Trivy or Syft formats, lacking delta optimisation. +* Few products expose **Policy‑as‑Code** with full UI editing **and** history audit in a single package. +* None address quota‑aware throttling without hidden paywalls. + +**Stella Ops** fills this gap by combining *speed*, *modular openness*, *sovereign readiness* **and transparent quota limits**—making thorough supply‑chain security attainable for every team, not just cloud‑native startups. + +--- +*Last updated: 14 Jul 2025* diff --git a/docs/02_WHY.md b/docs/02_WHY.md new file mode 100755 index 00000000..21b83e94 --- /dev/null +++ b/docs/02_WHY.md @@ -0,0 +1,121 @@ +# 2 · WHY — Why Stella Ops Exists + +> Explaining the concrete pain we solve, why the world needs **one more** DevSecOps +> platform, and the success signals that prove we are on the right track. + +Software‑supply‑chain attacks, licence‑risk, and incomplete SBOM coverage slow +teams and compliance audits to a crawl. Most existing scanners: + +* **Assume Internet** access for CVE feeds or SaaS back‑ends. +* **Parse an entire image** every build (no layer‑delta optimisation). +* **Accept a single SBOM format** (usually Trivy JSON) and choke on anything else. +* Offer **no built‑in policy history / audit trail**. +* Require 30‑60 s wall‑time per scan, an order of magnitude slower than modern CI + expectations. +* **Hide quota limits** or throttle without warning once you move past free trials. + +--- +# 1 Free‑Tier Quota — Why **{{ quota_token }} **? + +The limit of **{{ quota_token }} SBOM scans per UTC day** was not chosen at random. + +| Constraint | Analysis | Outcome | +|------------|----------|---------| +| **SMB workload** | Internal survey across 37 SMBs shows median **210** container builds/day (p95 ≈ 290). | {{ quota_token }} gives ≈ 1.6 × head‑room without forcing a paid tier. | +| **Cost of feeds** | Hosting, Trivy DB mirrors & CVE merge traffic average **≈ $14 / 1 000 scans**. | {{ quota_token }} /day yields <$5 infra cost per user — sustainable for an OSS project. | +| **Incentive to upgrade** | Larger orgs (> 300 builds/day) gain ROI from Plus/Pro tiers anyway. | Clear upsell path without hurting hobbyists. | + +> **In one sentence:**  *{{ quota_token }} scans cover the daily needs of a typical small / +> medium business, keep free usage genuinely useful and still leave a financial +> runway for future development*. + +## 1.1 How the Quota Is Enforced (1‑minute view) + +* Backend loads the **Quota plug‑in** at startup. +* Every `/scan` call passes the caller’s **Client‑JWT** to the plug‑in. +* The plug‑in **increments a counter in Redis** under + `quota::` (expires at UTC midnight). +* Soft wait‑wall (5 s) after limit; hard wait‑wall (60 s) after 30 blocked calls. +* For **offline installs**, a *1‑month validity Client‑JWT* ships inside every + **Offline Update Kit (OUK)** tarball. Uploading the OUK refreshes the token + automatically. + +Detailed sequence living in **30_QUOTA_ENFORCEMENT_FLOW.md**. + + + +--- + +## 2 · Why *Another* DevSecOps Product? — Macro Drivers + +| Driver | Evidence | Implication for Tooling | +|--------|----------|-------------------------| +| **Exploding supply‑chain attacks** | Sonatype 2024 report shows **742 %** growth since 2020. | SBOMs & provenance checks must be default, not “best‑practice”. | +| **Regulation tsunami** | • US EO 14028 & NIST SP 800‑218
• EU Cyber‑Resilience Act (CRA) in force 2026
• Local critical‑infrastructure rules in some jurisdictions | Vendors must *attest* build provenance (SLSA) and store tamper‑proof SBOMs. | +| **Runtime‑cost intolerance** | Pipelines build hundreds of images/day; waiting > 10 s per scan breaks SLA. | Need **delta‑aware** engines that reuse layer analyses (< 1 s warm scans). | +| **Air‑gap & sovereignty demands** | Finance/defence prohibit outbound traffic; data must stay on‑prem. | Ship **self‑contained registry + CVE DB** and run offline. | +| **Predictable free‑tier limits** | Teams want clarity, not surprise throttling. | Provide **transparent {{ quota_token }} scans/day quota**, early banner & graceful wait‑wall. | + +> **Therefore:** The market demands a **modular, SBOM‑first, sub‑5 s, 100 % self‑hosted** +> platform **with a transparent free‑tier quota**—precisely the niche Stella Ops targets. + +--- + +## 3 · Gap in Current Tooling + +* Trivy / Syft create SBOMs but re‑analyse **every** layer → wasted minutes/day. +* Policy engines (OPA/Rego) are separate binaries, with no UI or change history. +* No mainstream OSS bundle ships an **anonymous internal registry** for air‑gapped pulls. +* Provenance attestation (SLSA) and Rekor transparency logs remain “bring‑your‑own”. +* Free tiers either stop at 100 scans **or** silently throttle; none announce a **clear {{ quota_token }} /day allowance**. + +--- + +## 4 · Why Stella Ops Can Win + +1. **Speed First** — Delta‑SBOM flow uses cached layers to hit `< 1 s` warm scans. +2. **Multi‑Format Ready** — Auto‑detects Trivy‑JSON, SPDX‑JSON, CycloneDX‑JSON; UI + lets teams choose per‑project defaults. +3. **Offline by Default** — Ships an **anonymous internal Docker registry** + (`StellaOps.Registry`) plus Redis, Mongo, CVE DB, and UI in a single compose up. +4. **Open & Modular** — .NET hot‑load plug‑ins (`StellaOpsAttestor`, future scanners) + under AGPL; anyone can extend. +5. **Policy as Code** — YAML rules today, upgrade path to OPA/Rego with history stored + in Mongo via `StellaOps.MutePolicies`. +6. **Sovereign‑Ready** — Russian‑language UI, local vulnerability mirrors, zero + telemetry by default. +7. **Honest Free‑tier Boundaries** — Clear **{{ quota_token }} scans/day** limit, early banner at 200 and predictable wait‑wall—no hidden throttling. + +--- + +## 5 · Success Criteria — Signals We Solve the Problem + +* **Performance:** P95 scan < 5 s on first pass; `< 1 s` for warm delta scans. +* **Compatibility:** SBOMs in at least three formats consumed by ≥ 3 downstream tools. +* **Adoption:** ≥ 1 000 reported installs & ≥ 2 000 binary downloads by Q2‑2026. +* **Compliance:** Positive audits referencing CRA / NIST / SLSA readiness. +* **Community:** ≥ 15 first‑time contributors merged per quarter by 2026. +* **Transparency:** 0 support tickets complaining about “mystery throttling”. + +--- + +## 6 · Non‑Goals (2025‑2027) + +* Multi‑tenant SaaS offering. +* Automatic “fix‑PR” generation (left to ecosystem). +* Windows container **scanning** (Windows *agents* are on the 12‑month roadmap). + +--- + +## 7 · Stakeholder Pain‑Point Recap + +| Persona | Pain Today | Stella Ops Solution | +|---------|------------|---------------------| +| **Dev** | “My CI fails for 45 s on every push.” | < 5 s initial, < 1 s warm scans. | +| **Sec‑Ops** | Separate tools for SBOM, policy, and audit. | Unified UI + YAML / Rego policies with history. | +| **Infra** | Internet‑blocked site; no public pulls allowed. | Offline compose bundle + internal registry. | +| **Compliance** | Need CRA‑ready provenance by 2026. | Future `StellaOpsAttestor` SLSA + Rekor integration. | +| **Budget owner** | Fears hidden overage charges in “free” tiers. | Transparent {{ quota_token }} scans/day limit, visible in UI/API. | + +--- +*Last updated: 14 Jul 2025 (sync with free‑tier quota rev 2.0).* diff --git a/docs/03_QUICKSTART.md b/docs/03_QUICKSTART.md new file mode 100755 index 00000000..900954cb --- /dev/null +++ b/docs/03_QUICKSTART.md @@ -0,0 +1,156 @@ +# Five‑Minute Quick‑Start ⚡ +Run your first container scan locally + +> **Heads‑up** – the public α `v0.1.0` image drops **late 2025**. +> Once it is published as +> `registry.stella-ops.org/stella-ops/stella-ops:0.1.0‑alpha` +> every command on this page works without changes. + +--- + +## 0 · What you need 🔧 + +| Requirement | Minimum | Notes | +|-------------|---------|-------| +| OS | Ubuntu 22.04 • Alma 9 | x86‑64 or arm64 | +| Docker | Engine 25 • Compose v2 | `docker -v` | +| CPU / RAM | 2 vCPU / 2 GiB | Dev‑laptop baseline | +| Disk | 10 GiB SSD | SBOM cache | + +> **Tip –** If you already have Redis & MongoDB, skip the infra +> compose file and point Stella Ops at those hosts via `.env`. + +--- + +## 1 · Fetch the signed Compose bundles 📦 + +```bash +# Infrastructure (Redis + MongoDB) +curl -LO https://get.stella-ops.org/docker-compose.infrastructure.yml +curl -LO https://get.stella-ops.org/docker-compose.infrastructure.yml.sig + +# Core scanner stack +curl -LO https://get.stella-ops.org/docker-compose.stella-ops.yml +curl -LO https://get.stella-ops.org/docker-compose.stella-ops.yml.sig + +# Verify signatures (supply‑chain 101) +cosign verify-blob --key https://stella-ops.org/keys/cosign.pub \ + --signature docker-compose.infrastructure.yml.sig docker-compose.infrastructure.yml +cosign verify-blob --key https://stella-ops.org/keys/cosign.pub \ + --signature docker-compose.stella-ops.yml.sig docker-compose.stella-ops.yml +```` + +--- + +## 2 · Create `.env` 🗝️ + +```bash + +# ─── Identity (shows in reports) ─────────────────────────── +STELLA_OPS_COMPANY_NAME="Acme Corp" +STELLA_OPS_ISSUER_EMAIL="ops@acme.example" +STELLA_OPS_DEFAULT_ADMIN_USERNAME="admin" +STELLA_OPS_DEFAULT_ADMIN_PASSWORD="changeme!" +STELLA_OPS_DEFAULT_JWT="" # or load it later with +# docker --env-file .env compose -f docker-compose.stella-ops.yml exec stella set-jwt + + +# ─── Database secrets ────────────────────────────────────── +MONGO_INITDB_ROOT_USERNAME=stella_admin +MONGO_INITDB_ROOT_PASSWORD=$(openssl rand -base64 18) +MONGO_URL=mongodb + +REDIS_PASSWORD=$(openssl rand -base64 18) +REDIS_URL=redis + + + +``` + +--- + +## 3 · Start the supporting services 🗄️ + +```bash +docker compose --env-file .env -f docker-compose.infrastructure.yml pull +docker compose --env-file .env -f docker-compose.infrastructure.yml up -d +``` + +--- + +## 4 · Launch Stella Ops 🚀 + +```bash +docker compose --env-file .env -f docker-compose.stella-ops.yml pull +docker compose --env-file .env -f docker-compose.stella-ops.yml up -d +``` + +*Point your browser at* **`https://:8443`** – the certificate is +self‑signed in the alpha. +Default credentials: **`admin / changeme`** (rotate immediately!). + +--- + +## 5 · Run a scan 🔍 + +```bash +docker compose --env-file .env -f docker-compose.stella-ops.yml \ + exec stella-ops stella scan alpine:3.20 +``` + +* First scan downloads CVE feeds (\~ 50 MB). +* Warm scans finish in **≈ 5 s** on a 4‑vCPU host thanks to the Δ‑SBOM engine. + +--- + +## 6 · Reload or add a token later 🔄 + +```bash +# After adding STELLA_JWT to .env … +docker compose --env-file .env -f docker-compose.stella-ops.yml \ + exec stella-ops stella jwt +``` + +*Anonymous mode* → **{{ quota_anon }} scans/day** +*Token mode* → **{{ quota_token }} scans/day** +At **10 % of the daily max** a polite reminder appears; after {{ quota_token }} the server applies a **soft 5 s back‑off** and may return **429 + Retry‑After** until the daily reset. + +--- + +## 7 · Typical next steps ➡️ + +| Task | Where to look | +| ---------------------------------------- | ------------------------------------------------------------------- | +| CI pipelines (GitHub / GitLab / Jenkins) | [`docs/ci/`](ci/) | +| Air‑gapped install | [Offline Update Kit](10_OFFLINE_KIT.md) | +| Feature overview | [20\_FEATURES.md](20_FEATURES.md) | +| Governance & licence | [`LICENSE.md`](LICENSE.md) • [`11_GOVERNANCE.md`](11_GOVERNANCE.md) | + +--- + +## 8 · Uninstall / cleanup 🧹 + +```bash +docker compose --env-file .env -f docker-compose.stella-ops.yml down -v +docker compose --env-file .env -f docker-compose.infrastructure.yml down -v +rm compose-*.yml compose-*.yml.sig .env +``` + +--- + +### Licence & provenance 📜 + +Stella Ops is **AGPL‑3.0‑or‑later**. Every release ships: + +* **Cosign‑signed** container images +* A full **SPDX 2.3** SBOM + +```bash +cosign verify \ + --key https://stella-ops.org/keys/cosign.pub \ + registry.stella-ops.org/stella-ops/stella-ops: +``` + +--- + +© 2025‑2026 Stella Ops – free / libre / open‑source. diff --git a/docs/03_VISION.md b/docs/03_VISION.md new file mode 100755 index 00000000..4508b0ea --- /dev/null +++ b/docs/03_VISION.md @@ -0,0 +1,99 @@ +#  3 · Product Vision — **Stella Ops** +*(v1.3 — 12 Jul 2025 · supersedes v1.2; expanded with ecosystem integration, refined metrics, and alignment to emerging trends)* + +--- + +##  0 Preamble + +This Vision builds on the purpose and gap analysis defined in **01 WHY**. +It paints a three‑year “north‑star” picture of success for the open‑source project and sets the measurable guard‑rails that every roadmap item must serve, while fostering ecosystem growth and adaptability to trends like SBOM mandates, AI‑assisted security **and transparent usage quotas**. + +--- + +##  1 North‑Star Vision Statement (2027) + +> *By mid‑2027, Stella Ops is the fastest, most‑trusted self‑hosted SBOM scanner. Developers expect vulnerability feedback in **five seconds or less**—even while the free tier enforces a transparent **{{ quota_token }} scans/day** limit with graceful waiting. The project thrives on a vibrant plug‑in marketplace, weekly community releases, transparent governance, and seamless integrations with major CI/CD ecosystems—while never breaking the five‑second promise.* + +--- + +##  2 Outcomes & Success Metrics + +| KPI (community‑centric) | Baseline Jul 2025 | Target Q2‑2026 | North‑Star 2027 | +| -------------------------------- | ----------------- | -------------- | --------------- | +| ⭐ Gitea / GitHub stars | 0 | 4 000 | 10 000 | +| Weekly active Docker pulls | 0 | 1 500 | 4 000 | +| P95 SBOM scan time (alpine) | 5 s | **≤ 5 s** | **≤ 4 s** | +| Free‑tier scan satisfaction* | n/a | ≥ 90 % | ≥ 95 % | +| First‑time‑contributor PRs / qtr | 0 | 15 | 30 | + +\*Measured via anonymous telemetry *opt‑in only*: ratio of successful scans to `429 QuotaExceeded` errors. + +--- + +##  3 Strategic Pillars + +1. **Speed First** – preserve the sub‑5 s P95 wall‑time; any feature that hurts it must ship behind a toggle or plug‑in. **Quota throttling must apply a soft 5 s delay first, so “speed first” remains true even at the limit.** +2. **Offline‑by‑Design** – every byte required to scan ships in public images; Internet access is optional. +3. **Modular Forever** – capabilities land as hot‑load plug‑ins; the monolith can split without rewrites. +4. **Community Ownership** – ADRs and governance decisions live in public; new maintainers elected by meritocracy. +5. **Zero‑Surprise Upgrades & Limits** – SemVer discipline; `main` is always installable; minor upgrades never break CI YAML **and free‑tier limits are clearly documented, with early UI warnings.** +6. **Ecosystem Harmony** – Prioritise integrations with popular OSS tools (e.g., Trivy extensions, BuildKit hooks) to lower adoption barriers. + +--- + +##  4 Road‑map Themes (18‑24 months) + +| Horizon | Theme | Example EPIC | +| ------------------ | ----------------------- | ---------------------------------------------------------------------------------------------------------------------------------- | +| **Q3‑2025** (3 mo) | **Core Stability & UX** | One‑command installer; dark‑mode UI; baseline SBOM scanning; **Free‑tier Quota Service ({{ quota_token }} scans/day, early banner, wait‑wall).** | +| 6–12 mo | *Extensibility* | Scan‑service micro‑split PoC; community plugin marketplace beta. | +| 12–18 mo | *Ecosystem* | Community plug‑in marketplace launch; integrations with Syft and Harbor. | +| 18–24 mo | *Resilience & Scale* | Redis Cluster auto‑sharding; AI‑assisted triage plugin framework. | + +*(Granular decomposition lives in 25_LEDGER.md.) + +--- + +##  5 Stakeholder Personas & Benefits + +| Persona | Core Benefit | +| --------------------- | ---------------------------------------------------------------- | +| Solo OSS maintainer | Laptop scans in **≤ 5 s**; zero cloud reliance. | +| CI Platform Engineer | Single‑binary backend + Redis; stable YAML integrations. | +| Security Auditor | AGPL code, traceable CVE sources, reproducible benchmarks. | +| Community Contributor | Plugin hooks and good‑first issues; merit‑based maintainer path. | +| Budget‑conscious Lead | Clear **{{ quota_token }} scans/day** allowance before upgrades are required. | + +(See **01 WHY §3** for detailed pain‑points & evidence.) + +--- + +##  6 Non‑Goals (2025‑2027) + +* Multi‑tenant SaaS offering. +* Automated “fix PR” generation. +* Proprietary compliance certifications (left to downstream distros). +* Windows **container** scanning (agents only). + +--- + +##  7 Review & Change Process + +* **Cadence:** product owner leads a public Vision review every **2 sprints (≈ 1 quarter)**. +* **Amendments:** material changes require PR labelled `type:vision` + two maintainer approvals. +* **Versioning:** bump patch for typo, minor for KPI tweak, major if North‑Star statement shifts. +* **Community Feedback:** Open GitHub Discussions for input; incorporate top‑voted suggestions quarterly. + +--- + +## 8 · Change Log + +| Version | Date | Note (high‑level) | +| ------- | ----------- | ----------------------------------------------------------------------------------------------------- | +| v1.4 | 14‑Jul‑2025 | First public revision reflecting quarterly roadmap & KPI baseline. | +| v1.3 | 12‑Jul‑2025 | Expanded ecosystem pillar, added metrics/integrations, refined non-goals, community persona/feedback. | +| v1.2 | 11‑Jul‑2025 | Restructured to link with WHY; merged principles into Strategic Pillars; added review §7 | +| v1.1 | 11‑Jul‑2025 | Original OSS‑only vision | +| v1.0 | 09‑Jul‑2025 | First public draft | + +*(End of Product Vision v1.3)* diff --git a/docs/04_FEATURE_MATRIX.md b/docs/04_FEATURE_MATRIX.md new file mode 100755 index 00000000..4e4efb70 --- /dev/null +++ b/docs/04_FEATURE_MATRIX.md @@ -0,0 +1,34 @@ +# 4 · Feature Matrix — **Stella Ops** +*(rev 2.0 · 14 Jul 2025)* + +| Category | Capability | Free Tier (≤ 333 scans / day) | Community Plug‑in | Commercial Add‑On | Notes / ETA | +| ---------------------- | ------------------------------------- | ----------------------------- | ----------------- | ------------------- | ------------------------------------------ | +| **SBOM Ingestion** | Trivy‑JSON, SPDX‑JSON, CycloneDX‑JSON | ✅ | — | — | Auto‑detect on upload | +| | **Delta‑SBOM Cache** | ✅ | — | — | Warm scans < 1 s | +| **Scanning** | CVE lookup via local DB | ✅ | — | — | Update job ships weekly feeds | +| | Licence‑risk detection | ⏳ (roadmap Q4‑2025) | — | — | SPDX licence list | +| **Policy Engine** | YAML rules | ✅ | — | — | In‑UI editor | +| | OPA / Rego | ⏳ (β Q1‑2026) | ✅ plug‑in | — | Plug‑in enables Rego | +| **Registry** | Anonymous internal registry | ✅ | — | — | `StellaOps.Registry` image | +| **Attestation** | Cosign signing | ⏳ (Q1‑2026) | — | — | Requires `StellaOpsAttestor` | +| | SLSA provenance v1.0 | — | — | ⏳ (commercial 2026) | Enterprise need | +| | Rekor transparency log | — | ✅ plug‑in | — | Air‑gap replica support | +| **Quota & Throttling** | {{ quota_token }} scans/day soft limit | ✅ | — | — | Yellow banner at 200, wait‑wall post‑limit | +| | Usage API (`/quota`) | ✅ | — | — | CI can poll remaining scans | +| **User Interface** | Dark / light mode | ✅ | — | — | Auto‑detect OS theme | +| | Additional locale (Cyrillic) | ✅ | — | — | Default if `Accept‑Language: bg` or any other | +| | Audit trail | ✅ | — | — | Mongo history | +| **Deployment** | Docker Compose bundle | ✅ | — | — | Single‑node | +| | Helm chart (K8s) | ✅ | — | — | Horizontal scaling | +| | High‑availability split services | — | — | ✅ (Add‑On) | HA Redis & Mongo | +| **Extensibility** | .NET hot‑load plug‑ins | ✅ | N/A | — | AGPL reference SDK | +| | Community plug‑in marketplace | — | ⏳ (β Q2‑2026) | — | Moderated listings | +| **Telemetry** | Opt‑in anonymous metrics | ✅ | — | — | Required for quota satisfaction KPI | +| **Quota & Tokens** | **Client‑JWT issuance** | ✅ (online 12 h token) | — | — | `/connect/token` | +| | **Offline Client‑JWT (30 d)** | ✅ via OUK | — | — | Refreshed monthly in OUK | + +> **Legend:** ✅ = Included ⏳ = Planned — = Not applicable +> Rows marked “Commercial Add‑On” are optional paid components shipping outside the AGPL‑core; everything else is FOSS. + +--- +*Last updated: 14 Jul 2025 (quota rev 2.0).* diff --git a/docs/05_ROADMAP.md b/docs/05_ROADMAP.md new file mode 100755 index 00000000..1a1402f3 --- /dev/null +++ b/docs/05_ROADMAP.md @@ -0,0 +1,6 @@ +# Road‑map + +Milestones are maintained on the project website. +👉  + +_This stub exists to satisfy historic links._ \ No newline at end of file diff --git a/docs/05_SYSTEM_REQUIREMENTS_SPEC.md b/docs/05_SYSTEM_REQUIREMENTS_SPEC.md new file mode 100755 index 00000000..ddaa6a26 --- /dev/null +++ b/docs/05_SYSTEM_REQUIREMENTS_SPEC.md @@ -0,0 +1,204 @@ +# SYSTEM REQUIREMENTS SPECIFICATION +Stella Ops · self‑hosted supply‑chain‑security platform + +> **Audience** – core maintainers and external contributors who need an +> authoritative checklist of *what* the software must do (functional +> requirements) and *how well* it must do it (non‑functional +> requirements). Implementation details belong in Module Specs +> or ADRs—**not here**. + +--- + +## 1 · Purpose & Scope + +This SRS defines everything the **v0.8‑beta** release of _Stella Ops_ must do, **including the Free‑tier daily quota of {{ quota_token }} SBOM scans per token**. +Scope includes core platform, CLI, UI, quota layer, and plug‑in host; commercial or closed‑source extensions are explicitly out‑of‑scope. + +--- + +## 2 · References + +* [02_WHY.md](02_WHY.md) – market gap & problem statement +* [03_VISION.md](03_VISION.md) – north‑star, KPIs, quarterly themes +* [07_HIGH_LEVEL_ARCHITECTURE.md](07_HIGH_LEVEL_ARCHITECTURE.md) – context & data flow diagrams +* [08_MODULE_SPECIFICATIONS.md](08_MODULE_SPECIFICATIONS.md) – component APIs & plug‑in contracts +* [09_API_CLI_REFERENCE.md](09_API_CLI_REFERENCE.md) – REST & CLI surface + +--- + +## 3 · Definitions & Acronyms + +| Term | Meaning | +|------|---------| +| **SBOM** | Software Bill of Materials | +| **Delta SBOM** | Partial SBOM covering only image layers not previously analysed | +| **Registry** | Anonymous, read‑only Docker Registry v2 hosted internally | +| **OPA** | Open Policy Agent (Rego policy engine) | +| **Muting Policy** | Rule that downgrades or ignores specific findings | +| **SLSA** | Supply‑chain Levels for Software Artifacts (provenance framework) | +| **Rekor** | Sigstore transparency log for signatures | + +--- + +## 4 · Overall System Description + +The platform consists of: + +* **Stella Ops Backend** – REST API, queue, policy engine, DB. +* **StellaOps.Registry** – internal container registry for agents. +* **Stella CLI** – extracts SBOMs; supports multi‑format & delta. +* **Zastava Agent** – enforcement hook for admission‑control scenarios. +* **Web UI** – React/Next.js SPA consuming backend APIs. +* **Plug‑ins** – hot‑load binaries extending scanners, attestations, etc. + +All services run in Docker Compose or Kubernetes with optional Internet +access. + +--- + +## 5 · Functional Requirements (FR) + +### 5.1 Core Scanning + +| ID | Requirement | Priority | Verification | +|----|-------------|----------|--------------| +| F‑1 | System SHALL ingest **Trivy‑JSON, SPDX‑JSON, CycloneDX‑JSON** files. | MUST | UT‑SBOM‑001 | +| F‑2 | System SHALL **auto‑detect** SBOM type when `sbomType` param omitted. | MUST | UT‑SBOM‑002 | +| F‑3 | System SHALL **cache analysed layers** and reuse them in subsequent scans. | MUST | IT‑CACHE‑001 | +| F‑4 | System SHALL **enforce a soft limit of {{ quota_token }} scans per token per UTC day**. | MUST | IT‑QUOTA‑001 | +| F‑4a | Remaining quota SHALL be **persisted in Redis** under key `quota::`. | MUST | UT‑QUOTA‑REDIS | +| F‑4b | Exhausted quota SHALL trigger **HTTP 429** with `Retry‑After` header (UTC midnight). | MUST | IT‑QUOTA‑002 | +| F‑4c | When quota is ≤ 40 % remaining, **UI banner** MUST turn yellow and show count‑down. | SHOULD | UI‑E2E‑005 | +| F‑4d | `/quota` endpoint SHALL return JSON `{"limit":{{ quota_token }} ,"remaining":N,"resetsAt":""}`. | SHOULD | API‑DOC‑003 | +| F‑5 | Policy engine SHALL evaluate **YAML rules** against scan results. | MUST | UT‑POL‑001 | +| F‑6 | Hot‑pluggable .NET plug‑ins SHALL be loadable **without service restart**. | MUST | IT‑PLUGIN‑001 | +| F‑7 | CLI (`stella scan`) SHOULD exit **non‑zero** when CVSS≥7 vulnerabilities found. | SHOULD | CL‑INT‑003 | +| *(… all previously documented F‑8 – F‑12 rows retained unchanged …)* | + + +### 5.2 Internal Docker Repository + +| Ref | Requirement | +|-----|-------------| +| **FR‑REPO‑1** | Platform SHALL include **StellaOps.Registry** exposing Docker Registry v2 API (ports 5000/443). | +| **FR‑REPO‑2** | Registry SHALL allow anonymous, *read‑only* pulls for at least three images:
• `stella/sbom‑builder`
• `stella/cli`
• `stella/zastava`. | +| **FR‑REPO‑3** | Registry MAY enable optional basic‑auth without code changes. | + +### 5.3 SBOM Generation & Handling + +| Ref | Requirement | +|-----|-------------| +| **FR‑SBOM‑1** | SBOM builder SHALL produce Trivy‑JSON **and** at least one additional format: SPDX‑JSON and CycloneDX‑JSON. | +| **FR‑SBOM‑2** | For every generated SBOM, builder SHALL create a side‑car file `.sbom.type` containing the format identifier. | +| **FR‑SBOM‑3** | Stella CLI SHALL read the `.sbom.type` file and include `sbomType` parameter when uploading. | +| **FR‑SBOM‑4** | Backend SHALL auto‑detect SBOM type when parameter is missing. | +| **FR‑SBOM‑5** | UI Settings SHALL expose a dropdown to select default SBOM format (system‑wide fallback). | + +#### 5.3.1 Delta SBOM (layer reuse) + +| Ref | Requirement | +|-----|-------------| +| **FR‑DELTA‑1** | Builder SHALL compute SHA256 digests of each image layer and POST array to `/layers/missing`; response time ≤ 20 ms (P95). | +| **FR‑DELTA‑2** | Builder SHALL generate SBOM **only** for layers returned as “missing”. | +| **FR‑DELTA‑3** | End‑to‑end warm scan time (image differing by ≤ 2 layers) SHALL be ≤ 1 s (P95). | + +### 5.4 Policy as Code (Muting & Expiration) + +| Ref | Requirement | +|-----|-------------| +| **FR‑POLICY‑1** | Backend SHALL store policies as YAML by default, convertible to Rego for advanced use‑cases. | +| **FR‑POLICY‑2** | Each policy change SHALL create an immutable history record (timestamp, actor, diff). | +| **FR‑POLICY‑3** | REST endpoints `/policy/import`, `/policy/export`, `/policy/validate` SHALL accept YAML or Rego payloads. | +| **FR‑POLICY‑4** | Web UI Policies tab SHALL provide Monaco editor with linting for YAML and Rego. | +| **FR‑POLICY‑5** | **StellaOps.MutePolicies** module SHALL expose CLI `stella policies apply --file scan‑policy.yaml`. | + +### 5.5 SLSA Attestations & Rekor (TODO > 6 mo) + +| Ref | Requirement | +|-----|-------------| +| **FR‑SLSA‑1** | **TODO** – Generate provenance in SLSA‑Provenance v0.2 for each SBOM. | +| **FR‑REKOR‑1** | **TODO** – Sign SBOM hashes and upload to local Rekor mirror; verify during scan. | + +### 5.6 CLI & API Interface + +| Ref | Requirement | +|-----|-------------| +| **FR‑CLI‑1** | CLI `stella scan` SHALL accept `--sbom-type {trivy,spdx,cyclonedx,auto}`. | +| **FR‑API‑1** | API `/scan` SHALL accept `sbomType` query/body field (optional). | +| **FR‑API‑2** | API `/layers/missing` SHALL accept JSON array of digests and return JSON array of missing digests. | + +--- + +## 6 · Non‑Functional Requirements (NFR) + +| Ref | Category | Requirement | +|-----|----------|-------------| +| **NFR‑PERF‑1** | Performance | P95 cold scan ≤ 5 s; warm ≤ 1 s (see **FR‑DELTA‑3**). | +| **NFR‑PERF‑2** | Throughput | System shall sustain 60 concurrent scans on 8‑core node without queue depth >10. | +| **NFR‑AVAIL‑1** | Availability | All services shall start offline; any Internet call must be optional. | +| **NFR‑SCAL‑1** | Scalability | Horizontal scaling via Kubernetes replicas for backend, Redis Sentinel, Mongo replica set. | +| **NFR‑SEC‑1** | Security | All inter‑service traffic shall use TLS or localhost sockets. | +| **NFR‑COMP‑1** | Compatibility | Platform shall run on x86‑64 Linux kernel ≥ 5.10; Windows agents (TODO > 6 mo) must support Server 2019+. | +| **NFR‑I18N‑1** | Internationalisation | UI must support EN and at least one additional locale (Cyrillic). | +| **NFR‑OBS‑1** | Observability | Export Prometheus metrics for scan duration, queue length, policy eval duration. | + +--- + +## 7 Acceptance Criteria + +1. Issue {{ quota_token }} `/scan` calls; next returns random slow down and `Retry‑After`. +2. Redis failure during test → API returns **0 remaining** & warns in logs. +3. UI banner activates at 133 remaining; clears next UTC midnight. + +--- +## 8 · System Interfaces + +### 8.1 External APIs + +*(This is the complete original table, plus new `/quota` row.)* + +| Path | Method | Auth | Quota | Description | +|------|--------|------|-------|-------------| +| `/scan` | POST | Bearer | ✅ | Submit SBOM or `imageRef` for scanning. | +| `/quota` | GET | Bearer | ❌ | Return remaining quota for current token. | +| `/policy/rules` | GET/PUT | Bearer+RBAC | ❌ | CRUD YAML or Rego policies. | +| `/plugins` | POST/GET | Bearer+Admin | ❌ | Upload or list plug‑ins. | + +```bash +GET /quota +Authorization: Bearer + +200 OK +{ +"limit": {{ quota_token }}, +"remaining": 121, +"resetsAt": "2025-07-14T23:59:59Z" +} +``` + +## 9 · Assumptions & Constraints + +* Hardware reference: 8 vCPU, 8 GB RAM, NVMe SSD. +* Mongo DB and Redis run co‑located unless horizontal scaling enabled. +* All docker images tagged `latest` are immutable (CI process locks digests). +* Rego evaluation runs in embedded OPA Go‑library (no external binary). + +--- + +## 10 · Future Work (Beyond 12 Months) + +* Rekor transparency log cross‑cluster replication. +* AI‑assisted false‑positive triage plug‑in. +* Cluster‑wide injection for live runtime scanning. + +--- + +## 11 · Revision History + +| Version | Date | Notes | +|---------|------|-------| +| **v1.2** | 11‑Jul‑2025 | Commercial references removed; plug‑in contract (§ 3.3) and new NFR categories added; added User Classes & Traceability. | +| v1.1 | 11‑Jul‑2025 | Split out RU‑specific items; OSS scope | +| v1.0 | 09‑Jul‑2025 | Original unified SRS | + +*(End of System Requirements Specification v1.2‑core)* diff --git a/docs/07_HIGH_LEVEL_ARCHITECTURE.md b/docs/07_HIGH_LEVEL_ARCHITECTURE.md new file mode 100755 index 00000000..fd8d5250 --- /dev/null +++ b/docs/07_HIGH_LEVEL_ARCHITECTURE.md @@ -0,0 +1,388 @@ +# 7 · High‑Level Architecture — **Stella Ops** + +--- + +## 0 Purpose & Scope + +Give contributors, DevOps engineers and auditors a **complete yet readable map** of the Core: + +* Major runtime components and message paths. +* Where plug‑ins, CLI helpers and runtime agents attach. +* Technology choices that enable the sub‑5 second SBOM goal. +* Typical operational scenarios (pipeline scan, mute, nightly re‑scan, etc.). + +Anything enterprise‑only (signed PDF, custom/regulated TLS, LDAP, enforcement) **must arrive as a plug‑in**; the Core never hard‑codes those concerns. +--- +## 1 Component Overview + +| # | Component | Responsibility | +|---|-----------|---------------| +| 1 | **API Gateway** | REST endpoints (`/scan`, `/quota`, **`/token/offline`**); token auth; quota enforcement | +| 2 | **Scan Service** | SBOM parsing, Delta‑SBOM cache, vulnerability lookup | +| 3 | **Policy Engine** | YAML / (optional) Rego rule evaluation; verdict assembly | +| 4 | **Quota Service** | Per‑token counters; **333 scans/day**; waits & HTTP 429 | +| 5 | **Client‑JWT Issuer** | Issues 30‑day offline tokens; bundles them into OUK | +| 5 | **Registry** | Anonymous internal Docker registry for agents, SBOM uploads | +| 6 | **Web UI** | React/Blazor SPA; dashboards, policy editor, quota banner | +| 7 | **Data Stores** | **Redis** (cache, quota) & **MongoDB** (SBOMs, findings, audit) | +| 8 | **Plugin Host** | Hot‑load .NET DLLs; isolates community plug‑ins | +| 9 | **Agents** | `sbom‑builder`, `Stella CLI` scanner CLI, future `StellaOpsAttestor` | + + +```mermaid +flowchart TD + subgraph "External Actors" + DEV["Developer / DevSecOps / Manager"] + CI["CI/CD Pipeline (e.g., Stella CLI)"] + K8S["Kubernetes Cluster (e.g., Zastava Agent)"] + end + + subgraph "Stella Ops Runtime" + subgraph "Core Services" + CORE["Stella Core
(REST + gRPC APIs, Orchestration)"] + REDIS[("Redis
(Cache, Queues, Trivy DB Mirror)")] + MONGO[("MongoDB
(Optional: Long-term Storage)")] + POL["Mute Policies
(OPA & YAML Evaluator)"] + REG["StellaOps Registry
(Docker Registry v2)"] + ATT["StellaOps Attestor
(SLSA + Rekor)"] + end + + subgraph "Agents & Builders" + SB["SBOM Builder
(Go Binary: Extracts Layers, Generates SBOMs)"] + SA["Stella CLI
(Pipeline Helper: Invokes Builder, Triggers Scans)"] + ZA["Zastava Agent
(K8s Webhook: Enforces Policies, Inventories Containers)"] + end + + subgraph "Scanners & UI" + TRIVY["Trivy Scanner
(Plugin Container: Vulnerability Scanning)"] + UI["Web UI
(Vue3 + Tailwind: Dashboards, Policy Editor)"] + CLI["Stella CLI
(CLI Helper: Triggers Scans, Mutes)"] + end + end + + DEV -->|Browses Findings, Mutes CVEs| UI + DEV -->|Triggers Scans| CLI + CI -->|Generates SBOM, Calls /scan| SA + K8S -->|Inventories Containers, Enforces Gates| ZA + + UI -- "REST" --> CORE + CLI -- "REST/gRPC" --> CORE + SA -->|Scan Requests| CORE + SB -->|Uploads SBOMs| CORE + ZA -->|Policy Gates| CORE + + CORE -- "Queues, Caches" --> REDIS + CORE -- "Persists Data" --> MONGO + CORE -->|Evaluates Policies| POL + CORE -->|Attests Provenance| ATT + CORE -->|Scans Vulnerabilities| TRIVY + + SB -- "Pulls Images" --> REG + SA -- "Pulls Images" --> REG + ZA -- "Pulls Images" --> REG + + style DEV fill:#f9f,stroke:#333 + style CI fill:#f9f,stroke:#333 + style K8S fill:#f9f,stroke:#333 + style CORE fill:#ddf,stroke:#333 + style REDIS fill:#fdd,stroke:#333 + style MONGO fill:#fdd,stroke:#333 + style POL fill:#dfd,stroke:#333 + style REG fill:#dfd,stroke:#333 + style ATT fill:#dfd,stroke:#333 + style SB fill:#fdf,stroke:#333 + style SA fill:#fdf,stroke:#333 + style ZA fill:#fdf,stroke:#333 + style TRIVY fill:#ffd,stroke:#333 + style UI fill:#ffd,stroke:#333 + style CLI fill:#ffd,stroke:#333 +``` + +* **Developer / DevSecOps / Manager** – browses findings, mutes CVEs, triggers scans. +* **Stella CLI** – generates SBOMs and calls `/scan` during CI. +* **Zastava Agent** – inventories live containers; Core ships it in *passive* mode only (no kill). + +### 1.1 Client‑JWT Lifecycle (offline aware) + +1. **Online instance** – user signs in → `/connect/token` issues JWT valid 12 h. +2. **Offline instance** – JWT with `exp ≈ 30 days` ships in OUK; backend + **re‑signs** and stores it during import. +3. Tokens embed a `tier` claim (“Free”) and `maxScansPerDay: 333`. +4. On expiry the UI surfaces a red toast **7 days** in advance. + +--- + +## 2 · Component Responsibilities (runtime view) + +| Component | Core Responsibility | Implementation Highlights | +| -------------------------- | ---------------------------------------------------------------------------------------------------------- | --------------------------------------------------------- | +| **Stella Core** | Orchestrates scans, persists SBOM blobs, serves REST/gRPC APIs, fans out jobs to scanners & policy engine. | .NET {{ dotnet }}, CQRS, Redis Streams; pluggable runner interfaces. | +| **SBOM Builder** | Extracts image layers, queries Core for *missing* layers, generates SBOMs (multi‑format), uploads blobs. | Go binary; wraps Trivy & Syft libs. | +| **Stella CLI** | Pipeline‑side helper; invokes Builder, triggers scan, streams progress back to CI/CD. | Static musl build. | +| **Zastava Agent** | K8s admission webhook enforcing policy verdicts before Pod creation. | Rust for sub‑10 ms latencies. | +| **UI** | Angular 17 SPA for dashboards, settings, policy editor. | Tailwind CSS; Webpack module federation (future). | +| **Redis** | Cache, queue, Trivy‑DB mirror, layer diffing. | Single instance or Sentinel. | +| **MongoDB** (opt.) | Long‑term SBOM & policy audit storage (> 180 days). | Optional; enabled via flag. | +| **StellaOps.Registry** | Anonymous read‑only Docker v2 registry with optional Cosign verification. | `registry :2` behind nginx reverse proxy. | +| **StellaOps.MutePolicies** | YAML/Rego evaluator, policy version store, `/policy/*` API. | Embeds OPA‑WASM; falls back to `opa exec`. | +| **StellaOpsAttestor** | Generate SLSA provenance & Rekor signatures; verify on demand. | Side‑car container; DSSE + Rekor CLI. | + +All cross‑component calls use dependency‑injected interfaces—no +intra‑component reach‑ins. + +--- + +## 3 · Principal Backend Modules & Plug‑in Hooks + +| Namespace | Responsibility | Built‑in Tech / Default | Plug‑in Contract | +| --------------- | -------------------------------------------------- | ----------------------- | ------------------------------------------------- | +| `configuration` | Parse env/JSON, health‑check endpoint | .NET {{ dotnet }} Options | `IConfigValidator` | +| `identity` | Embedded OAuth2/OIDC (OpenIddict 6) | MIT OpenIddict | `IIdentityProvider` for LDAP/SAML/JWT gateway | +| `pluginloader` | Discover DLLs, SemVer gate, optional Cosign verify | Reflection + Cosign | `IPluginLifecycleHook` for telemetry | +| `scanning` | SBOM‑ & image‑flow orchestration; runner pool | Trivy CLI (default) | `IScannerRunner` – e.g., Grype, Copacetic, Clair | +| `feedser` (vulnerability ingest/merge/export service) | Nightly NVD merge & feed enrichment | Hangfire job | drop-in `*.Schedule.dll` for OSV, GHSA, NVD 2.0, CNNVD, CNVD, ENISA, JVN and BDU feeds | +| `tls` | TLS provider abstraction | OpenSSL | `ITlsProvider` for custom suites (incl. **SM2**, where law or security requires it) | +| `reporting` | Render HTML/PDF reports | RazorLight | `IReportRenderer` | +| `ui` | Angular SPA & i18n | Angular {{ angular }} | new locales via `/locales/{lang}.json` | +| `scheduling` | Cron + retries | Hangfire | any recurrent job via `*.Schedule.dll` | + +```mermaid +classDiagram + class configuration + class identity + class pluginloader + class scanning + class feedser + class tls + class reporting + class ui + class scheduling + + class AllModules + + configuration ..> identity : Uses + identity ..> pluginloader : Authenticates Plugins + pluginloader ..> scanning : Loads Scanner Runners + scanning ..> feedser : Triggers Feed Merges + tls ..> AllModules : Provides TLS Abstraction + reporting ..> ui : Renders Reports for UI + scheduling ..> feedser : Schedules Nightly Jobs + + note for scanning "Pluggable: ISScannerRunner
e.g., Trivy, Grype" + note for feedser "Pluggable: *.Schedule.dll
e.g., OSV, GHSA Feeds" + note for identity "Pluggable: IIdentityProvider
e.g., LDAP, SAML" + note for reporting "Pluggable: IReportRenderer
e.g., Custom PDF" +``` + +**When remaining = 0:** +API returns `429 Too Many Requests`, `Retry‑After: ` (sequence omitted for brevity). + +--- + +## 4 · Data Flows + +### 4.1 SBOM‑First (≤ 5 s P95) + +Builder produces SBOM locally, so Core never touches the Docker +socket. +Trivy path hits ≤ 5 s on alpine:3.19 with warmed DB. +Image‑unpack fallback stays ≤ 10 s for 200 MB images. + +```mermaid +sequenceDiagram + participant CI as CI/CD Pipeline (Stella CLI) + participant SB as SBOM Builder + participant CORE as Stella Core + participant REDIS as Redis Queue + participant RUN as Scanner Runner (e.g., Trivy) + participant POL as Policy Evaluator + + CI->>SB: Invoke SBOM Generation + SB->>CORE: Check Missing Layers (/layers/missing) + CORE->>REDIS: Query Layer Diff (SDIFF) + REDIS-->>CORE: Missing Layers List + CORE-->>SB: Return Missing Layers + SB->>SB: Generate Delta SBOM + SB->>CORE: Upload SBOM Blob (POST /scan(sbom)) + CORE->>REDIS: Enqueue Scan Job + REDIS->>RUN: Fan Out to Runner + RUN->>RUN: Perform Vulnerability Scan + RUN-->>CORE: Return Scan Results + CORE->>POL: Evaluate Mute Policies + POL-->>CORE: Policy Verdict + CORE-->>CI: JSON Verdict & Progress Stream + Note over CORE,CI: Achieves ≤5s P95 with Warmed DB +``` + +### 4.2 Delta SBOM + +Builder collects layer digests. +`POST /layers/missing` → Redis SDIFF → missing layer list (< 20 ms). +SBOM generated only for those layers and uploaded. + +### 4.3 Feedser Harvest & Export + +```mermaid +sequenceDiagram + participant SCHED as Feedser Scheduler + participant CONN as Source Connector Plug-in + participant FEEDSER as Feedser Core + participant MONGO as MongoDB (Canonical Advisories) + participant EXPORT as Exporter (JSON / Trivy DB) + participant ART as Artifact Store / Offline Kit + + SCHED->>CONN: Trigger window (init/resume) + CONN->>CONN: Fetch source documents + metadata + CONN->>FEEDSER: Submit raw document for parsing + FEEDSER->>FEEDSER: Parse & normalize to DTO + FEEDSER->>FEEDSER: Merge & deduplicate canonical advisory + FEEDSER->>MONGO: Write advisory, provenance, merge_event + FEEDSER->>EXPORT: Queue export delta request + EXPORT->>MONGO: Read canonical snapshot/deltas + EXPORT->>EXPORT: Build deterministic JSON & Trivy DB artifacts + EXPORT->>ART: Publish artifacts / Offline Kit bundle + ART-->>FEEDSER: Record export state + digests +``` + +### 4.4 Identity & Auth Flow + +OpenIddict issues JWTs via client‑credentials or password grant. +An IIdentityProvider plug‑in can delegate to LDAP, SAML or external OIDC +without Core changes. +--- +## 5 · Runtime Helpers + +| Helper | Form | Purpose | Extensible Bits | +|-----------|---------------------------------------|--------------------------------------------------------------------|-------------------------------------------| +| **Stella CLI** | Distroless CLI | Generates SBOM, calls `/scan`, honours threshold flag | `--engine`, `--pdf-out` piped to plug‑ins | +| **Zastava** | Static Go binary / DaemonSet | Watches Docker/CRI‑O events; uploads SBOMs; can enforce gate | Policy plug‑in could alter thresholds | + +--- + +## 6 · Persistence & Cache Strategy + +| Store | Primary Use | Why chosen | +|----------------|-----------------------------------------------|--------------------------------| +| **MongoDB** | Feedser canonical advisories, merge events, export state | Deterministic canonical store with flexible schema | +| **Redis 7** | CLI quotas, short-lived job scheduling, layer diff cache | Sub-1 ms P99 latency for hot-path coordination | +| **Local tmpfs**| Trivy layer cache (`/var/cache/trivy`) | Keeps disk I/O off hot path | + +```mermaid +flowchart LR + subgraph "Persistence Layers" + REDIS[(Redis: Quotas & Short-lived Queues
Sub-1ms P99)] + MONGO[(MongoDB: Canonical Advisories
Merge Events & Export State)] + TMPFS[(Local tmpfs: Trivy Layer Cache
Low I/O Overhead)] + end + + CORE["Stella Core"] -- Queues & SBOM Cache --> REDIS + CORE -- Long-term Storage --> MONGO + TRIVY["Trivy Scanner"] -- Layer Unpack Cache --> TMPFS + + style REDIS fill:#fdd,stroke:#333 + style MONGO fill:#dfd,stroke:#333 + style TMPFS fill:#ffd,stroke:#333 +``` + +--- + +## 7 · Typical Scenarios + +| # | Flow | Steps | +|---------|----------------------------|-------------------------------------------------------------------------------------------------| +| **S‑1** | Pipeline Scan & Alert | Stella CLI → SBOM → `/scan` → policy verdict → CI exit code & link to *Scan Detail* | +| **S‑2** | Mute Noisy CVE | Dev toggles **Mute** in UI → rule stored in Redis → next build passes | +| **S‑3** | Nightly Re‑scan | `SbomNightly.Schedule` re‑queues SBOMs (mask‑filter) → dashboard highlights new Criticals | +| **S‑4** | Feed Update Cycle | `Feedser (vulnerability ingest/merge/export service)` refreshes feeds → UI *Feed Age* tile turns green | +| **S‑5** | Custom Report Generation | Plug‑in registers `IReportRenderer` → `/report/custom/{digest}` → CI downloads artifact | + +```mermaid +sequenceDiagram + participant DEV as Developer + participant UI as Web UI + participant CORE as Stella Core + participant REDIS as Redis + participant RUN as Scanner Runner + + DEV->>UI: Toggle Mute for CVE + UI->>CORE: Update Mute Rule (POST /policy/mute) + CORE->>REDIS: Store Mute Policy + Note over CORE,REDIS: YAML/Rego Evaluator Updates + + alt Next Pipeline Build + CI->>CORE: Trigger Scan (POST /scan) + CORE->>RUN: Enqueue & Scan + RUN-->>CORE: Raw Findings + CORE->>REDIS: Apply Mute Policies + REDIS-->>CORE: Filtered Verdict (Passes) + CORE-->>CI: Success Exit Code + end +``` + +```mermaid +sequenceDiagram + participant CRON as SbomNightly.Schedule + participant CORE as Stella Core + participant REDIS as Redis Queue + participant RUN as Scanner Runner + participant UI as Dashboard + + CRON->>CORE: Re-queue SBOMs (Mask-Filter) + CORE->>REDIS: Enqueue Filtered Jobs + REDIS->>RUN: Fan Out to Runners + RUN-->>CORE: New Scan Results + CORE->>UI: Highlight New Criticals + Note over CORE,UI: Focus on Changes Since Last Scan +``` +--- + +## 8 · UI Fast Facts + +* **Stack** – Angular 17 + Vite dev server; Tailwind CSS. +* **State** – Signals + RxJS for live scan progress. +* **i18n / l10n** – JSON bundles served from `/locales/{lang}.json`. +* **Module Structure** – Lazy‑loaded feature modules (`dashboard`, `scans`, `settings`); runtime route injection by UI plug‑ins (road‑map Q2‑2026). + +--- + +## 9 · Cross‑Cutting Concerns + +* **Security** – containers run non‑root, `CAP_DROP:ALL`, read‑only FS, hardened seccomp profiles. +* **Observability** – Serilog JSON, OpenTelemetry OTLP exporter, Prometheus `/metrics`. +* **Upgrade Policy** – `/api/v1` endpoints & CLI flags stable across a minor; breaking changes bump major. + +--- + +## 10 · Performance & Scalability + +| Scenario | P95 target | Bottleneck | Mitigation | +|-----------------|-----------:|-----------------|-------------------------------------------------| +| SBOM‑first | ≤ 5 s | Redis queue | More CPU, increase `ScannerPool.Workers` | +| Image‑unpack | ≤ 10 s | Layer unpack | Prefer SBOM path, warm Docker cache | +| High concurrency| 40 rps | Runner CPU | Scale Core replicas + side‑car scanner services | + +--- + +## 11 · Future Architectural Anchors + +* **ScanService micro‑split (gRPC)** – isolate heavy runners for large clusters. +* **UI route plug‑ins** – dynamic Angular module loader (road‑map Q2‑2026). +* **Redis Cluster** – transparently sharded cache once sustained > 100 rps. + +--- + +## 12 · Assumptions & Trade‑offs + +Requires Docker/CRI‑O runtime; .NET 9 available on hosts; Windows containers are out‑of‑scope this cycle. +Embedded auth simplifies deployment but may need plug‑ins for enterprise IdPs. +Speed is prioritised over exhaustive feature parity with heavyweight commercial scanners. + +--- + +## 13 · References & Further Reading + +* **C4 Model** – +* **.NET Architecture Guides** – +* **OSS Examples** – Kubernetes Architecture docs, Prometheus design papers, Backstage. + +*(End of High‑Level Architecture v2.2)* diff --git a/docs/08_MODULE_SPECIFICATIONS.md b/docs/08_MODULE_SPECIFICATIONS.md new file mode 100755 index 00000000..c9c0d236 --- /dev/null +++ b/docs/08_MODULE_SPECIFICATIONS.md @@ -0,0 +1,208 @@ +# 8 · Detailed Module Specifications — **Stella Ops Feedser** +_This document describes the Feedser service, its supporting libraries, connectors, exporters, and test assets that live in the OSS repository._ + +--- + +## 0 Scope + +Feedser is the vulnerability ingest/merge/export subsystem of Stella Ops. It +fetches primary advisories, normalizes and deduplicates them into MongoDB, and +produces deterministic JSON and Trivy DB exports. This document lists the +projects that make up that workflow, the extension points they expose, and the +artefacts they ship. + +--- + +## 1 Repository layout (current) + +```text +src/ + ├─ Directory.Build.props / Directory.Build.targets + ├─ StellaOps.Plugin/ + ├─ StellaOps.Feedser.Core/ + ├─ StellaOps.Feedser.Core.Tests/ + ├─ StellaOps.Feedser.Models/ (+ .Tests/) + ├─ StellaOps.Feedser.Normalization/ (+ .Tests/) + ├─ StellaOps.Feedser.Merge/ (+ .Tests/) + ├─ StellaOps.Feedser.Storage.Mongo/ (+ .Tests/) + ├─ StellaOps.Feedser.Exporter.Json/ (+ .Tests/) + ├─ StellaOps.Feedser.Exporter.TrivyDb/ (+ .Tests/) + ├─ StellaOps.Feedser.Source.* / StellaOps.Feedser.Source.*.Tests/ + ├─ StellaOps.Feedser.Testing/ + ├─ StellaOps.Feedser.Tests.Shared/ + ├─ StellaOps.Feedser.WebService/ (+ .Tests/) + ├─ PluginBinaries/ + └─ StellaOps.Feedser.sln +``` + +Each folder is a .NET project (or set of projects) referenced by +`StellaOps.Feedser.sln`. Build assets are shared through the root +`Directory.Build.props/targets` so conventions stay consistent. + +--- + +## 2 Shared libraries + +| Project | Purpose | Key extension points | +|---------|---------|----------------------| +| `StellaOps.Plugin` | Base contracts for connectors, exporters, and DI routines plus Cosign validation helpers. | `IFeedConnector`, `IExporterPlugin`, `IDependencyInjectionRoutine` | +| `StellaOps.DependencyInjection` | Composable service registrations for Feedser and plug-ins. | `IDependencyInjectionRoutine` discovery | +| `StellaOps.Feedser.Testing` | Common fixtures, builders, and harnesses for integration/unit tests. | `FeedserMongoFixture`, test builders | +| `StellaOps.Feedser.Tests.Shared` | Shared assembly metadata and fixtures wired in via `Directory.Build.props`. | Test assembly references | + +--- + +## 3 Core projects + +| Project | Responsibility | Extensibility | +|---------|----------------|---------------| +| `StellaOps.Feedser.WebService` | ASP.NET Core minimal API hosting Feedser jobs, status endpoints, and scheduler. | DI-based plug-in discovery; configuration binding | +| `StellaOps.Feedser.Core` | Job orchestration, connector pipelines, merge workflows, export coordination. | `IFeedConnector`, `IExportJob`, deterministic merge policies | +| `StellaOps.Feedser.Models` | Canonical advisory DTOs and enums persisted in MongoDB and exported artefacts. | Partial classes for source-specific metadata | +| `StellaOps.Feedser.Normalization` | Version comparison, CVSS normalization, text utilities for canonicalization. | Helpers consumed by connectors/merge | +| `StellaOps.Feedser.Merge` | Precedence evaluation, alias graph maintenance, merge-event hashing. | Policy extensions via DI | +| `StellaOps.Feedser.Storage.Mongo` | Repository layer for documents, DTOs, advisories, merge events, export state. | Connection string/config via options | +| `StellaOps.Feedser.Exporter.Json` | Deterministic vuln-list JSON export pipeline. | Dependency injection for storage + plugin to host | +| `StellaOps.Feedser.Exporter.TrivyDb` | Builds Trivy DB artefacts from canonical advisories. | Optional ORAS push routines | + +### 3.1 StellaOps.Feedser.WebService + +* Hosts minimal API endpoints (`/health`, `/status`, `/jobs`). +* Runs the scheduler that triggers connectors and exporters according to + configured windows. +* Applies dependency-injection routines from `PluginBinaries/` at startup only + (restart-time plug-ins). + +### 3.2 StellaOps.Feedser.Core + +* Defines job primitives (fetch, parse, map, merge, export) used by connectors. +* Coordinates deterministic merge flows and writes `merge_event` documents. +* Provides telemetry/log scopes consumed by WebService and exporters. + +### 3.3 StellaOps.Feedser.Storage.Mongo + +* Persists raw documents, DTO records, canonical advisories, aliases, affected + packages, references, merge events, export state, and job leases. +* Exposes repository helpers for exporters to stream full/delta snapshots. + +### 3.4 StellaOps.Feedser.Exporter.* + +* `Exporter.Json` mirrors the Aqua vuln-list tree with canonical ordering. +* `Exporter.TrivyDb` builds Trivy DB Bolt archives and optional OCI bundles. +* Both exporters honour deterministic hashing and respect export cursors. + +--- + +## 4 Source connectors + +Connectors live under `StellaOps.Feedser.Source.*` and conform to the interfaces +in `StellaOps.Plugin`. + +| Family | Project(s) | Notes | +|--------|------------|-------| +| Distro PSIRTs | `StellaOps.Feedser.Source.Distro.*` | Debian, Red Hat, SUSE, Ubuntu connectors with NEVRA/EVR helpers. | +| Vendor PSIRTs | `StellaOps.Feedser.Source.Vndr.*` | Adobe, Apple, Cisco, Chromium, Microsoft, Oracle, VMware. | +| Regional CERTs | `StellaOps.Feedser.Source.Cert*`, `Source.Ru.*`, `Source.Ics.*`, `Source.Kisa` | Provide enrichment metadata while preserving vendor precedence. | +| OSS ecosystems | `StellaOps.Feedser.Source.Ghsa`, `Source.Osv`, `Source.Cve`, `Source.Kev`, `Source.Acsc`, `Source.Cccs`, `Source.Jvn` | Emit SemVer/alias-rich advisories. | + +Each connector ships fixtures/tests under the matching `*.Tests` project. + +--- + +## 5 · Module Details + +> _Focus on the Feedser-specific services that replace the legacy FeedMerge cron._ + +### 5.1 Feedser.Core + +* Owns the fetch → parse → merge → export job pipeline and enforces deterministic + merge hashes (`merge_event`). +* Provides `JobSchedulerBuilder`, job coordinator, and telemetry scopes consumed + by the WebService and exporters. + +### 5.2 Feedser.Storage.Mongo + +* Bootstrapper creates collections/indexes (documents, dto, advisory, alias, + affected, merge_event, export_state, jobs, locks). +* Repository APIs surface full/delta advisory reads for exporters, plus + SourceState and job lease persistence. + +### 5.3 Feedser.Exporter.Json / Feedser.Exporter.TrivyDb + +* JSON exporter mirrors vuln-list layout with per-file digests and manifest. +* Trivy DB exporter shells or native-builds Bolt archives, optionally pushes OCI + layers, and records export cursors. Delta runs reuse unchanged blobs from the + previous full baseline, annotating `metadata.json` with `mode`, `baseExportId`, + `baseManifestDigest`, `resetBaseline`, and `delta.changedFiles[]`/`delta.removedPaths[]`. + ORAS pushes honour `publishFull` / `publishDelta`, and offline bundles respect + `includeFull` / `includeDelta` for air-gapped syncs. + +### 5.4 Feedser.WebService + +* Minimal API host exposing `/health`, `/ready`, `/jobs` and wiring telemetry. +* Loads restart-time plug-ins from `PluginBinaries/`, executes Mongo bootstrap, + and registers built-in connectors/exporters with the scheduler. + +### 5.5 Plugin host & DI bridge + +* `StellaOps.Plugin` + `StellaOps.DependencyInjection` provide the contracts and + helper routines for connectors/exporters to integrate with the WebService. + +--- + +## 6 · Plug-ins & Agents + +* **Plug-in discovery** – restart-only; the WebService enumerates + `PluginBinaries/` (or configured directories) and executes the contained + `IDependencyInjectionRoutine` implementations. +* **Connector/exporter packages** – each source/exporter can ship as a plug-in + assembly with its own options and HttpClient configuration, keeping the core + image minimal. +* **StellaOps CLI (agent)** – new `StellaOps.Cli` module that exposes + `scanner`, `scan`, and `db` verbs (via System.CommandLine 2.0) to download + scanner container bundles, install them locally, execute scans against target + directories, automatically upload results, and trigger Feedser jobs (`db + fetch/merge/export`) aligned with the SBOM-first workflow described in + `AGENTS.md`. +* **Offline Kit** – bundles Feedser plug-ins, JSON tree, Trivy DB, and export + manifests so air-gapped sites can load the latest vulnerability data without + outbound connectivity. + +--- + +## 7 · Docker & Distribution Artefacts + +| Artefact | Path / Identifier | Notes | +|----------|-------------------|-------| +| Feedser WebService image | `containers/feedser/Dockerfile` (built via CI) | Self-contained ASP.NET runtime hosting scheduler/endpoints. | +| Plugin bundle | `PluginBinaries/` | Mounted or baked-in assemblies for connectors/exporters. | +| Offline Kit tarball | Produced by CI release pipeline | Contains JSON tree, Trivy DB OCI layout, export manifest, and plug-ins. | +| Local dev compose | `scripts/` + future compose overlays | Developers can run MongoDB, Redis (optional), and WebService locally. | + +--- + +## 8 · Performance Budget + +| Scenario | Budget | Source | +|----------|--------|--------| +| Advisory upsert (large advisory) | ≤ 500 ms/advisory | `AdvisoryStorePerformanceTests` (Mongo) | +| Advisory fetch (`GetRecent`) | ≤ 200 ms/advisory | Same performance test harness | +| Advisory point lookup (`Find`) | ≤ 200 ms/advisory | Same performance test harness | +| Bulk upsert/fetch cycle | ≤ 28 s total for 30 large advisories | Same performance test harness | +| Feedser job scheduling | Deterministic cron execution via `JobSchedulerHostedService` | `StellaOps.Feedser.Core` tests | +| Trivy DB export | Deterministic digests across runs (ongoing TODO for end-to-end test) | `Exporter.TrivyDb` backlog | + +Budgets are enforced in automated tests where available; outstanding TODO/DOING +items (see task boards) continue tracking gaps such as exporter determinism. + +--- + +## 9 Testing + +* Unit and integration tests live alongside each component (`*.Tests`). +* Shared fixtures come from `StellaOps.Feedser.Testing` and + `StellaOps.Feedser.Tests.Shared` (linked via `Directory.Build.props`). +* Integration suites use ephemeral MongoDB and Redis via Testcontainers to + validate end-to-end flow without external dependencies. + +--- diff --git a/docs/09_API_CLI_REFERENCE.md b/docs/09_API_CLI_REFERENCE.md new file mode 100755 index 00000000..483e2310 --- /dev/null +++ b/docs/09_API_CLI_REFERENCE.md @@ -0,0 +1,461 @@ +# API & CLI Reference + +*Purpose* – give operators and integrators a single, authoritative spec for REST/GRPC calls **and** first‑party CLI tools (`stella-cli`, `zastava`, `stella`). +Everything here is *source‑of‑truth* for generated Swagger/OpenAPI and the `--help` screens in the CLIs. + +--- + +## 0 Quick Glance + +| Area | Call / Flag | Notes | +| ------------------ | ------------------------------------------- | ------------------------------------------------------------------------------ | +| Scan entry | `POST /scan` | Accepts SBOM or image; sub‑5 s target | +| Delta check | `POST /layers/missing` | <20 ms reply; powers *delta SBOM* feature | +| Rate‑limit / quota | — | Headers **`X‑Stella‑Quota‑Remaining`**, **`X‑Stella‑Reset`** on every response | +| Policy I/O | `GET /policy/export`, `POST /policy/import` | YAML now; Rego coming | +| Policy lint | `POST /policy/validate` | Returns 200 OK if ruleset passes | +| Auth | `POST /connect/token` (OpenIddict) | Client‑credentials preferred | +| Health | `GET /healthz` | Simple liveness probe | +| Attestation * | `POST /attest` (TODO Q1‑2026) | SLSA provenance + Rekor log | +| CLI flags | `--sbom-type` `--delta` `--policy-file` | Added to `stella` | + +\* Marked **TODO** → delivered after sixth month (kept on Feature Matrix “To Do” list). + +--- + +## 1 Authentication + +Stella Ops uses **OAuth 2.0 / OIDC** (token endpoint mounted via OpenIddict). + +``` +POST /connect/token +Content‑Type: application/x-www-form-urlencoded + +grant_type=client_credentials& +client_id=ci‑bot& +client_secret=REDACTED& +scope=stella.api +``` + +Successful response: + +```json +{ + "access_token": "eyJraWQi...", + "token_type": "Bearer", + "expires_in": 3600 +} +``` + +> **Tip** – pass the token via `Authorization: Bearer ` on every call. + +--- + +## 2 REST API + +### 2.0 Obtain / Refresh Offline‑Token + +```text +POST /token/offline +Authorization: Bearer +``` + +| Body field | Required | Example | Notes | +|------------|----------|---------|-------| +| `expiresDays` | no | `30` | Max 90 days | + +```json +{ + "jwt": "eyJhbGciOiJSUzI1NiIsInR5cCI6...", + "expires": "2025‑08‑17T00:00:00Z" +} +``` + +Token is signed with the backend’s private key and already contains +`"maxScansPerDay": {{ quota_token }}`. + + +### 2.1 Scan – Upload SBOM **or** Image + +``` +POST /scan +``` + +| Param / Header | In | Required | Description | +| -------------------- | ------ | -------- | --------------------------------------------------------------------- | +| `X‑Stella‑Sbom‑Type` | header | no | `trivy-json-v2`, `spdx-json`, `cyclonedx-json`; omitted ➞ auto‑detect | +| `?threshold` | query | no | `low`, `medium`, `high`, `critical`; default **critical** | +| body | body | yes | *Either* SBOM JSON *or* Docker image tarball/upload URL | + +Every successful `/scan` response now includes: + +| Header | Example | +|--------|---------| +| `X‑Stella‑Quota‑Remaining` | `129` | +| `X‑Stella‑Reset` | `2025‑07‑18T23:59:59Z` | +| `X‑Stella‑Token‑Expires` | `2025‑08‑17T00:00:00Z` | + +**Response 200** (scan completed): + +```json +{ + "digest": "sha256:…", + "summary": { + "Critical": 0, + "High": 3, + "Medium": 12, + "Low": 41 + }, + "policyStatus": "pass", + "quota": { + "remaining": 131, + "reset": "2025-07-18T00:00:00Z" + } +} +``` + +**Response 202** – queued; polling URL in `Location` header. + +--- + +### 2.2 Delta SBOM – Layer Cache Check + +``` +POST /layers/missing +Content‑Type: application/json +Authorization: Bearer +``` + +```json +{ + "layers": [ + "sha256:d38b...", + "sha256:af45..." + ] +} +``` + +**Response 200** — <20 ms target: + +```json +{ + "missing": [ + "sha256:af45..." + ] +} +``` + +Client then generates SBOM **only** for the `missing` layers and re‑posts `/scan`. + +--- + +### 2.3 Policy Endpoints + +| Method | Path | Purpose | +| ------ | ------------------ | ------------------------------------ | +| `GET` | `/policy/export` | Download live YAML ruleset | +| `POST` | `/policy/import` | Upload YAML or Rego; replaces active | +| `POST` | `/policy/validate` | Lint only; returns 400 on error | +| `GET` | `/policy/history` | Paginated change log (audit trail) | + +```yaml +# Example import payload (YAML) +version: "1.0" +rules: + - name: Ignore Low dev + severity: [Low, None] + environments: [dev, staging] + action: ignore +``` + +Validation errors come back as: + +```json +{ + "errors": [ + { + "path": "$.rules[0].severity", + "msg": "Invalid level 'None'" + } + ] +} +``` + +--- + +### 2.4 Attestation (Planned – Q1‑2026) + +``` +POST /attest +``` + +| Param | Purpose | +| ----------- | ------------------------------------- | +| body (JSON) | SLSA v1.0 provenance doc | +| | Signed + stored in local Rekor mirror | + +Returns `202 Accepted` and `Location: /attest/{id}` for async verify. + +--- + +## 3 StellaOps CLI (`stellaops-cli`) + +The new CLI is built on **System.CommandLine 2.0.0‑beta5** and mirrors the Feedser backend REST API. +Configuration follows the same precedence chain everywhere: + +1. Environment variables (e.g. `API_KEY`, `STELLAOPS_BACKEND_URL`, `StellaOps:ApiKey`) +2. `appsettings.json` → `appsettings.local.json` +3. `appsettings.yaml` → `appsettings.local.yaml` +4. Defaults (`ApiKey = ""`, `BackendUrl = ""`, cache folders under the current working directory) + +**Authority auth client resilience settings** + +| Setting | Environment variable | Default | Purpose | +|---------|----------------------|---------|---------| +| `StellaOps:Authority:Resilience:EnableRetries` | `STELLAOPS_AUTHORITY_ENABLE_RETRIES` | `true` | Toggle Polly wait-and-retry handlers for discovery/token calls | +| `StellaOps:Authority:Resilience:RetryDelays` | `STELLAOPS_AUTHORITY_RETRY_DELAYS` | `1s,2s,5s` | Comma/space-separated backoff sequence (HH:MM:SS) | +| `StellaOps:Authority:Resilience:AllowOfflineCacheFallback` | `STELLAOPS_AUTHORITY_ALLOW_OFFLINE_CACHE_FALLBACK` | `true` | Reuse cached discovery/JWKS metadata when Authority is temporarily unreachable | +| `StellaOps:Authority:Resilience:OfflineCacheTolerance` | `STELLAOPS_AUTHORITY_OFFLINE_CACHE_TOLERANCE` | `00:10:00` | Additional tolerance window added to the discovery/JWKS cache lifetime | + +See `docs/dev/32_AUTH_CLIENT_GUIDE.md` for recommended profiles (online vs. air-gapped) and testing guidance. + +| Command | Purpose | Key Flags / Arguments | Notes | +|---------|---------|-----------------------|-------| +| `stellaops-cli scanner download` | Fetch and install scanner container | `--channel ` (default `stable`)
`--output `
`--overwrite`
`--no-install` | Saves artefact under `ScannerCacheDirectory`, verifies digest/signature, and executes `docker load` unless `--no-install` is supplied. | +| `stellaops-cli scan run` | Execute scanner container against a directory (auto-upload) | `--target ` (required)
`--runner ` (default from config)
`--entry `
`[scanner-args...]` | Runs the scanner, writes results into `ResultsDirectory`, emits a structured `scan-run-*.json` metadata file, and automatically uploads the artefact when the exit code is `0`. | +| `stellaops-cli scan upload` | Re-upload existing scan artefact | `--file ` | Useful for retries when automatic upload fails or when operating offline. | +| `stellaops-cli db fetch` | Trigger connector jobs | `--source ` (e.g. `redhat`, `osv`)
`--stage ` (default `fetch`)
`--mode ` | Translates to `POST /jobs/source:{source}:{stage}` with `trigger=cli` | +| `stellaops-cli db merge` | Run canonical merge reconcile | — | Calls `POST /jobs/merge:reconcile`; exit code `0` on acceptance, `1` on failures/conflicts | +| `stellaops-cli db export` | Kick JSON / Trivy exports | `--format ` (default `json`)
`--delta`
`--publish-full/--publish-delta`
`--bundle-full/--bundle-delta` | Sets `{ delta = true }` parameter when requested and can override ORAS/bundle toggles per run | +| `stellaops-cli auth ` | Manage cached tokens for StellaOps Authority | `auth login --force` (ignore cache)
`auth status`
`auth whoami` | Uses `StellaOps.Auth.Client`; honours `StellaOps:Authority:*` configuration, stores tokens under `~/.stellaops/tokens` by default, and `whoami` prints subject/scope/expiry | + +When running on an interactive terminal without explicit override flags, the CLI uses Spectre.Console prompts to let you choose per-run ORAS/offline bundle behaviour. +| `stellaops-cli config show` | Display resolved configuration | — | Masks secret values; helpful for air‑gapped installs | + +**Logging & exit codes** + +- Structured logging via `Microsoft.Extensions.Logging` with single-line console output (timestamps in UTC). +- `--verbose / -v` raises log level to `Debug`. +- Command exit codes bubble up: backend conflict → `1`, cancelled via `CTRL+C` → `130`, scanner exit codes propagate as-is. + +**Artifact validation** + +- Downloads are verified against the `X-StellaOps-Digest` header (SHA-256). When `StellaOps:ScannerSignaturePublicKeyPath` points to a PEM-encoded RSA key, the optional `X-StellaOps-Signature` header is validated as well. +- Metadata for each bundle is written alongside the artefact (`*.metadata.json`) with digest, signature, source URL, and timestamps. +- Retry behaviour is controlled via `StellaOps:ScannerDownloadAttempts` (default **3** with exponential backoff). +- Successful `scan run` executions create timestamped JSON artefacts inside `ResultsDirectory` plus a `scan-run-*.json` metadata envelope documenting the runner, arguments, timing, and stdout/stderr. The artefact is posted back to Feedser automatically. + +#### Trivy DB export metadata (`metadata.json`) + +`stellaops-cli db export --format trivy-db` (and the backing `POST /jobs/export:trivy-db`) always emits a `metadata.json` document in the OCI layout root. Operators consuming the bundle or delta updates should inspect the following fields: + +| Field | Type | Purpose | +| ----- | ---- | ------- | +| `mode` | `full` \| `delta` | Indicates whether the current run rebuilt the entire database (`full`) or only the changed files (`delta`). | +| `baseExportId` | string? | Export ID of the last full baseline that the delta builds upon. Only present for `mode = delta`. | +| `baseManifestDigest` | string? | SHA-256 digest of the manifest belonging to the baseline OCI layout. | +| `resetBaseline` | boolean | `true` when the exporter rotated the baseline (e.g., repo change, delta chain reset). Treat as a full refresh. | +| `treeDigest` | string | Canonical SHA-256 digest of the JSON tree used to build the database. | +| `treeBytes` | number | Total bytes across exported JSON files. | +| `advisoryCount` | number | Count of advisories included in the export. | +| `exporterVersion` | string | Version stamp of `StellaOps.Feedser.Exporter.TrivyDb`. | +| `builder` | object? | Raw metadata emitted by `trivy-db build` (version, update cadence, etc.). | +| `delta.changedFiles[]` | array | Present when `mode = delta`. Each entry lists `{ "path": "", "length": , "digest": "sha256:..." }`. | +| `delta.removedPaths[]` | array | Paths that existed in the previous manifest but were removed in the new run. | + +When the planner opts for a delta run, the exporter copies unmodified blobs from the baseline layout identified by `baseManifestDigest`. Consumers that cache OCI blobs only need to fetch the `changedFiles` and the new manifest/metadata unless `resetBaseline` is true. +When pushing to ORAS, set `feedser:exporters:trivyDb:oras:publishFull` / `publishDelta` to control whether full or delta runs are copied to the registry. Offline bundles follow the analogous `includeFull` / `includeDelta` switches under `offlineBundle`. + +Example configuration (`appsettings.yaml`): + +```yaml +feedser: + exporters: + trivyDb: + oras: + enabled: true + publishFull: true + publishDelta: false + offlineBundle: + enabled: true + includeFull: true + includeDelta: false +``` + + +**Authentication** + +- API key is sent as `Authorization: Bearer ` automatically when configured. +- Anonymous operation is permitted only when Feedser runs with + `authority.allowAnonymousFallback: true`. This flag is temporary—plan to disable + it before **2025-12-31 UTC** so bearer tokens become mandatory. + +Authority-backed auth workflow: +1. Configure Authority settings via config or env vars (see sample below). Minimum fields: `Url`, `ClientId`, and either `ClientSecret` (client credentials) or `Username`/`Password` (password grant). +2. Run `stellaops-cli auth login` to acquire and cache a token. Use `--force` if you need to ignore an existing cache entry. +3. Execute CLI commands as normal—the backend client injects the cached bearer token automatically and retries on transient 401/403 responses with operator guidance. +4. Inspect the cache with `stellaops-cli auth status` (shows expiry, scope, mode) or clear it via `stellaops-cli auth logout`. +5. Run `stellaops-cli auth whoami` to dump token subject, audience, issuer, scopes, and remaining lifetime (verbose mode prints additional claims). +6. Expect Feedser to emit audit logs for each `/jobs*` request showing `subject`, + `clientId`, `scopes`, `status`, and whether network bypass rules were applied. + +Tokens live in `~/.stellaops/tokens` unless `StellaOps:Authority:TokenCacheDirectory` overrides it. Cached tokens are reused offline until they expire; the CLI surfaces clear errors if refresh fails. + +**Configuration file template** + +```jsonc +{ + "StellaOps": { + "ApiKey": "your-api-token", + "BackendUrl": "https://feedser.example.org", + "ScannerCacheDirectory": "scanners", + "ResultsDirectory": "results", + "DefaultRunner": "docker", + "ScannerSignaturePublicKeyPath": "", + "ScannerDownloadAttempts": 3, + "Authority": { + "Url": "https://authority.example.org", + "ClientId": "feedser-cli", + "ClientSecret": "REDACTED", + "Username": "", + "Password": "", + "Scope": "feedser.jobs.trigger", + "TokenCacheDirectory": "" + } + } +} +``` + +Drop `appsettings.local.json` or `.yaml` beside the binary to override per environment. + +--- + +### 2.5 Misc Endpoints + +| Path | Method | Description | +| ---------- | ------ | ---------------------------- | +| `/healthz` | GET | Liveness; returns `"ok"` | +| `/metrics` | GET | Prometheus exposition (OTel) | +| `/version` | GET | Git SHA + build date | + +--- + +## 3 First‑Party CLI Tools + +### 3.1 `stella` + +> *Package SBOM + Scan + Exit code* – designed for CI. + +``` +Usage: stella [OPTIONS] IMAGE_OR_SBOM +``` + +| Flag / Option | Default | Description | +| --------------- | ----------------------- | -------------------------------------------------- | +| `--server` | `http://localhost:8080` | API root | +| `--token` | *env `STELLA_TOKEN`* | Bearer token | +| `--sbom-type` | *auto* | Force `trivy-json-v2`/`spdx-json`/`cyclonedx-json` | +| `--delta` | `false` | Enable delta layer optimisation | +| `--policy-file` | *none* | Override server rules with local YAML/Rego | +| `--threshold` | `critical` | Fail build if ≥ level found | +| `--output-json` | *none* | Write raw scan result to file | +| `--wait-quota` | `true` | If 429 received, automatically wait `Retry‑After` and retry once. | + +**Exit codes** + +| Code | Meaning | +| ---- | ------------------------------------------- | +| 0 | Scan OK, policy passed | +| 1 | Vulnerabilities ≥ threshold OR policy block | +| 2 | Internal error (network etc.) | + +--- + +### 3.2 `stella‑zastava` + +> *Daemon / K8s DaemonSet* – watch container runtime, push SBOMs. + +Core flags (excerpt): + +| Flag | Purpose | +| ---------------- | ---------------------------------- | +| `--mode` | `listen` (default) / `enforce` | +| `--filter-image` | Regex; ignore infra/busybox images | +| `--threads` | Worker pool size | + +--- + +### 3.3 `stellopsctl` + +> *Admin utility* – policy snapshots, feed status, user CRUD. + +Examples: + +``` +stellopsctl policy export > policies/backup-2025-07-14.yaml +stellopsctl feed refresh # force OSV merge +stellopsctl user add dev-team --role developer +``` + +--- + +## 4 Error Model + +Uniform problem‑details object (RFC 7807): + +```json +{ + "type": "https://stella-ops.org/probs/validation", + "title": "Invalid request", + "status": 400, + "detail": "Layer digest malformed", + "traceId": "00-7c39..." +} +``` + +--- + +## 5 Rate Limits + +Default **40 requests / second / token**. +429 responses include `Retry-After` seconds header. + +--- + +## 6 FAQ & Tips + +* **Skip SBOM generation in CI** – supply a *pre‑built* SBOM and add `?sbom-only=true` to `/scan` for <1 s path. +* **Air‑gapped?** – point `--server` to `http://oukgw:8080` inside the Offline Update Kit. +* **YAML vs Rego** – YAML simpler; Rego unlocks time‑based logic (see samples). +* **Cosign verify plug‑ins** – enable `SCANNER_VERIFY_SIG=true` env to refuse unsigned plug‑ins. + +--- + +## 7 Planned Changes (Beyond 6 Months) + +These stay in *Feature Matrix → To Do* until design is frozen. + +| Epic / Feature | API Impact Sketch | +| ---------------------------- | ---------------------------------- | +| **SLSA L1‑L3** attestation | `/attest` (see §2.4) | +| Rekor transparency log | `/rekor/log/{id}` (GET) | +| Plug‑in Marketplace metadata | `/plugins/market` (catalog) | +| Horizontal scaling controls | `POST /cluster/node` (add/remove) | +| Windows agent support | Update LSAPI to PDE, no API change | + +--- + +## 8 References + +* OpenAPI YAML → `/openapi/v1.yaml` (served by backend) +* OAuth2 spec: +* SLSA spec: + +--- + +## 9 Changelog (truncated) + +* **2025‑07‑14** – added *delta SBOM*, policy import/export, CLI `--sbom-type`. +* **2025‑07‑12** – initial public reference. + +--- diff --git a/docs/10_FEEDSER_CLI_QUICKSTART.md b/docs/10_FEEDSER_CLI_QUICKSTART.md new file mode 100644 index 00000000..7283d6d8 --- /dev/null +++ b/docs/10_FEEDSER_CLI_QUICKSTART.md @@ -0,0 +1,289 @@ +# 10 · Feedser + CLI Quickstart + +This guide walks through configuring the Feedser web service and the `stellaops-cli` +tool so an operator can ingest advisories, merge them, and publish exports from a +single workstation. It focuses on deployment-facing surfaces only (configuration, +runtime wiring, CLI usage) and leaves connector/internal customization for later. + +--- + +## 0 · Prerequisites + +- .NET SDK **10.0.100-preview** (matches `global.json`) +- MongoDB instance reachable from the host (local Docker or managed) +- `trivy-db` binary on `PATH` for Trivy exports (and `oras` if publishing to OCI) +- Plugin assemblies present in `PluginBinaries/` (already included in the repo) +- Optional: Docker/Podman runtime if you plan to run scanners locally + +> **Tip** – air-gapped installs should preload `trivy-db` and `oras` binaries into the +> runner image since Feedser never fetches them dynamically. + +--- + +## 1 · Configure Feedser + +1. Copy the sample config to the expected location (CI/CD pipelines can stamp values + into this file during deployment—see the “Deployment automation” note below): + + ```bash + mkdir -p etc + cp etc/feedser.yaml.sample etc/feedser.yaml + ``` + +2. Edit `etc/feedser.yaml` and update the MongoDB DSN (and optional database name). + The default template configures plug-in discovery to look in `PluginBinaries/` + and disables remote telemetry exporters by default. + +3. (Optional) Override settings via environment variables. All keys are prefixed with + `FEEDSER_`. Example: + + ```bash + export FEEDSER_STORAGE__DSN="mongodb://user:pass@mongo:27017/feedser" + export FEEDSER_TELEMETRY__ENABLETRACING=false + ``` + +4. Start the web service from the repository root: + + ```bash + dotnet run --project src/StellaOps.Feedser.WebService + ``` + + On startup Feedser validates the options, boots MongoDB indexes, loads plug-ins, + and exposes: + + - `GET /health` – returns service status and telemetry settings + - `GET /ready` – performs a MongoDB `ping` + - `GET /jobs` + `POST /jobs/{kind}` – inspect and trigger connector/export jobs + + > **Security note** – authentication now ships via StellaOps Authority. Keep + > `authority.allowAnonymousFallback: true` only during the staged rollout and + > disable it before **2025-12-31 UTC** so tokens become mandatory. + +### Authority companion configuration (preview) + +1. Copy the Authority sample configuration: + + ```bash + cp etc/authority.yaml.sample etc/authority.yaml + ``` + +2. Update the issuer URL, token lifetimes, and plug-in descriptors to match your + environment. Authority expects per-plugin manifests in `etc/authority.plugins/`; + sample `standard.yaml` and `ldap.yaml` files are provided as starting points. + For air-gapped installs keep the default plug-in binary directory + (`../PluginBinaries/Authority`) so packaged plug-ins load without outbound access. + +3. Environment variables prefixed with `STELLAOPS_AUTHORITY_` override individual + fields. Example: + + ```bash + export STELLAOPS_AUTHORITY__ISSUER="https://authority.stella-ops.local" + export STELLAOPS_AUTHORITY__PLUGINDIRECTORIES__0="/srv/authority/plugins" + ``` + +--- + +## 2 · Configure the CLI + +The CLI reads configuration from JSON/YAML files *and* environment variables. The +defaults live in `src/StellaOps.Cli/appsettings.json` and expect overrides at runtime. + +| Setting | Environment variable | Default | Purpose | +| ------- | -------------------- | ------- | ------- | +| `BackendUrl` | `STELLAOPS_BACKEND_URL` | _empty_ | Base URL of the Feedser web service | +| `ApiKey` | `API_KEY` | _empty_ | Reserved for legacy key auth; leave empty when using Authority | +| `ScannerCacheDirectory` | `STELLAOPS_SCANNER_CACHE_DIRECTORY` | `scanners` | Local cache folder | +| `ResultsDirectory` | `STELLAOPS_RESULTS_DIRECTORY` | `results` | Where scan outputs are written | +| `Authority.Url` | `STELLAOPS_AUTHORITY_URL` | _empty_ | StellaOps Authority issuer/token endpoint | +| `Authority.ClientId` | `STELLAOPS_AUTHORITY_CLIENT_ID` | _empty_ | Client identifier for the CLI | +| `Authority.ClientSecret` | `STELLAOPS_AUTHORITY_CLIENT_SECRET` | _empty_ | Client secret (omit when using username/password grant) | +| `Authority.Username` | `STELLAOPS_AUTHORITY_USERNAME` | _empty_ | Username for password grant flows | +| `Authority.Password` | `STELLAOPS_AUTHORITY_PASSWORD` | _empty_ | Password for password grant flows | +| `Authority.Scope` | `STELLAOPS_AUTHORITY_SCOPE` | `feedser.jobs.trigger` | OAuth scope requested for backend operations | +| `Authority.TokenCacheDirectory` | `STELLAOPS_AUTHORITY_TOKEN_CACHE_DIR` | `~/.stellaops/tokens` | Directory that persists cached tokens | +| `Authority.Resilience.EnableRetries` | `STELLAOPS_AUTHORITY_ENABLE_RETRIES` | `true` | Toggle Polly retry handler for Authority HTTP calls | +| `Authority.Resilience.RetryDelays` | `STELLAOPS_AUTHORITY_RETRY_DELAYS` | `1s,2s,5s` | Comma- or space-separated backoff delays (hh:mm:ss) | +| `Authority.Resilience.AllowOfflineCacheFallback` | `STELLAOPS_AUTHORITY_ALLOW_OFFLINE_CACHE_FALLBACK` | `true` | Allow CLI to reuse cached discovery/JWKS metadata when Authority is offline | +| `Authority.Resilience.OfflineCacheTolerance` | `STELLAOPS_AUTHORITY_OFFLINE_CACHE_TOLERANCE` | `00:10:00` | Additional tolerance window applied to cached metadata | + +Example bootstrap: + +```bash +export STELLAOPS_BACKEND_URL="http://localhost:5000" +export STELLAOPS_RESULTS_DIRECTORY="$HOME/.stellaops/results" +export STELLAOPS_AUTHORITY_URL="https://authority.local" +export STELLAOPS_AUTHORITY_CLIENT_ID="feedser-cli" +export STELLAOPS_AUTHORITY_CLIENT_SECRET="s3cr3t" +dotnet run --project src/StellaOps.Cli -- db merge + +# Acquire a bearer token and confirm cache state +dotnet run --project src/StellaOps.Cli -- auth login +dotnet run --project src/StellaOps.Cli -- auth status +dotnet run --project src/StellaOps.Cli -- auth whoami +``` + +Refer to `docs/dev/32_AUTH_CLIENT_GUIDE.md` for deeper guidance on tuning retry/offline settings and rollout checklists. + +To persist configuration, you can create `stellaops-cli.yaml` next to the binary or +rely on environment variables for ephemeral runners. + +--- + +## 3 · Operating Workflow + +1. **Trigger connector fetch stages** + + ```bash + dotnet run --project src/StellaOps.Cli -- db fetch --source osv --stage fetch + dotnet run --project src/StellaOps.Cli -- db fetch --source osv --stage parse + dotnet run --project src/StellaOps.Cli -- db fetch --source osv --stage map + ``` + + Use `--mode resume` when continuing from a previous window: + + ```bash + dotnet run --project src/StellaOps.Cli -- db fetch --source redhat --stage fetch --mode resume + ``` + +2. **Merge canonical advisories** + + ```bash + dotnet run --project src/StellaOps.Cli -- db merge + ``` + +3. **Produce exports** + + ```bash + # JSON tree (vuln-list style) + dotnet run --project src/StellaOps.Cli -- db export --format json + + # Trivy DB (delta example) + dotnet run --project src/StellaOps.Cli -- db export --format trivy-db --delta + ``` + + Feedser always produces a deterministic OCI layout. The first run after a clean + bootstrap emits a **full** baseline; subsequent `--delta` runs reuse the previous + baseline’s blobs when only JSON manifests change. If the exporter detects that a + prior delta is still active (i.e., `LastDeltaDigest` is recorded) it automatically + upgrades the next run to a full export and resets the baseline so operators never + chain deltas indefinitely. The CLI exposes `--publish-full/--publish-delta` (for + ORAS pushes) and `--include-full/--include-delta` (for offline bundles) should you + need to override the defaults interactively. + + **Smoke-check delta reuse:** after the first baseline completes, run the export a + second time with `--delta` and verify that the new directory reports `mode=delta` + while reusing the previous layer blob. + + ```bash + export_root=${FEEDSER_EXPORT_ROOT:-exports/trivy} + base=$(ls -1d "$export_root"/* | sort | tail -n2 | head -n1) + delta=$(ls -1d "$export_root"/* | sort | tail -n1) + + jq -r '.mode,.baseExportId' "$delta/metadata.json" + + base_manifest=$(jq -r '.manifests[0].digest' "$base/index.json") + delta_manifest=$(jq -r '.manifests[0].digest' "$delta/index.json") + printf 'baseline manifest: %s\ndelta manifest: %s\n' "$base_manifest" "$delta_manifest" + + layer_digest=$(jq -r '.layers[0].digest' "$base/blobs/sha256/${base_manifest#sha256:}") + cmp "$base/blobs/sha256/${layer_digest#sha256:}" \ + "$delta/blobs/sha256/${layer_digest#sha256:}" + ``` + + `cmp` returning exit code `0` confirms the delta export reuses the baseline’s + `db.tar.gz` layer instead of rebuilding it. + +4. **Manage scanners (optional)** + + ```bash + dotnet run --project src/StellaOps.Cli -- scanner download --channel stable + dotnet run --project src/StellaOps.Cli -- scan run --entry scanners/latest/Scanner.dll --target ./sboms + dotnet run --project src/StellaOps.Cli -- scan upload --file results/scan-001.json + ``` + +Add `--verbose` to any command for structured console logs. All commands honour +`Ctrl+C` cancellation and exit with non-zero status codes when the backend returns +a problem document. + +--- + +## 4 · Verification Checklist + +- Feedser `/health` returns `"status":"healthy"` and Storage bootstrap is marked + complete after startup. +- CLI commands return HTTP 202 with a `Location` header (job tracking URL) when + triggering Feedser jobs. +- Export artefacts are materialised under the configured output directories and + their manifests record digests. +- MongoDB contains the expected `document`, `dto`, `advisory`, and `export_state` + collections after a run. + +--- + +## 5 · Deployment Automation + +- Treat `etc/feedser.yaml.sample` as the canonical template. CI/CD should copy it to + the deployment artifact and replace placeholders (DSN, telemetry endpoints, cron + overrides) with environment-specific secrets. +- Keep secret material (Mongo credentials, OTLP tokens) outside of the repository; + inject them via secret stores or pipeline variables at stamp time. +- When building container images, include `trivy-db` (and `oras` if used) so air-gapped + clusters do not need outbound downloads at runtime. + +--- + +## 5 · Next Steps + +- Enable authority-backed authentication in non-production first. Set + `authority.enabled: true` while keeping `authority.allowAnonymousFallback: true` + to observe logs, then flip it to `false` before 2025-12-31 UTC to enforce tokens. +- Automate the workflow above via CI/CD (compose stack or Kubernetes CronJobs). +- Pair with the Feedser connector teams when enabling additional sources so their + module-specific requirements are pulled in safely. + +--- + +## 6 · Authority Integration + +- Feedser now authenticates callers through StellaOps Authority using OAuth 2.0 + resource server flows. Populate the `authority` block in `feedser.yaml`: + + ```yaml + authority: + enabled: true + allowAnonymousFallback: false # keep true only during the staged rollout window + issuer: "https://authority.example.org" + audiences: + - "api://feedser" + requiredScopes: + - "feedser.jobs.trigger" + clientId: "feedser-jobs" + clientSecretFile: "../secrets/feedser-jobs.secret" + clientScopes: + - "feedser.jobs.trigger" + bypassNetworks: + - "127.0.0.1/32" + - "::1/128" + ``` + +- Store the client secret outside of source control. Either provide it via + `authority.clientSecret` (environment variable `FEEDSER_AUTHORITY__CLIENTSECRET`) + or point `authority.clientSecretFile` to a file mounted at runtime. +- Cron jobs running on the same host can keep using the API thanks to the loopback + bypass mask. Add additional CIDR ranges as needed; every bypass is logged. +- Export the same configuration to Kubernetes or systemd by setting environment + variables such as: + + ```bash + export FEEDSER_AUTHORITY__ENABLED=true + export FEEDSER_AUTHORITY__ALLOWANONYMOUSFALLBACK=false + export FEEDSER_AUTHORITY__ISSUER="https://authority.example.org" + export FEEDSER_AUTHORITY__CLIENTID="feedser-jobs" + export FEEDSER_AUTHORITY__CLIENTSECRETFILE="/var/run/secrets/feedser/authority-client" + ``` + +- CLI commands already pass `Authorization` headers when credentials are supplied. + Configure the CLI with matching Authority settings (`docs/09_API_CLI_REFERENCE.md`) + so that automation can obtain tokens with the same client credentials. Feedser + logs every job request with the client ID, subject (if present), scopes, and + a `bypass` flag so operators can audit cron traffic. diff --git a/docs/10_OFFLINE_KIT.md b/docs/10_OFFLINE_KIT.md new file mode 100755 index 00000000..4a9c9d9c --- /dev/null +++ b/docs/10_OFFLINE_KIT.md @@ -0,0 +1,139 @@ +# Offline Update Kit (OUK) — 100 % Air‑Gap Operation + +> **Status:** ships together with the public α `v0.1.0` (ETA **late 2025**). +> All commands below assume the bundle name +> `stella-ouk‑2025‑α.tar.gz` – adjust once the real date tag is known. + +--- + +## 1 · What’s in the bundle 📦 + +| Item | Purpose | +|------|---------| +| **Vulnerability database** | Pre‑merged snapshot of NVD 2.0, OSV, GHSA
+ optional **regional catalogue** feeds | +| **Container images** | Scanner + Zastava for **x86‑64** & **arm64** | +| **Cosign signatures** | Release attestation & SBOM integrity | +| **SPDX SBOM** | Cryptographically signed bill of materials | +| **Import manifest** | Check‑sums & version metadata | + +Nightly **delta patches** keep the bundle < 350 MB while staying *T‑1 day* +current. + +--- + +## 2 · Download & verify 🔒 + +```bash +curl -LO https://get.stella-ops.org/releases/latest/stella-ops-offline-usage-kit-v0.1a.tar.gz +curl -LO https://get.stella-ops.org/releases/latest/stella-ops-offline-usage-kit-v0.1a.tar.gz.sig + +cosign verify-blob \ + --key https://stella-ops.org/keys/cosign.pub \ + --signature stella-ops-offline-usage-kit-v0.1a.tar.gz.sig \ + stella-ops-offline-usage-kit-v0.1a.tar.gz +``` + +The output shows `Verified OK` and the SHA‑256 digest ‑ compare with the +release notes. + +--- + +## 3 · Import on the isolated host 🚀 + +```bash +docker compose --env-file .env -f compose-stella.yml \ + exec stella-ops stella ouk import stella-ops-offline-usage-kit-v0.1a.tar.gz +``` + +* The scanner verifies the Cosign signature **before** activation. +* DB switch is atomic – **no downtime** for running jobs. +* Import time on an SSD VM ≈ 5‑7 s. + +--- + +## 4 · How the quota works offline 🔢 + +| Mode | Daily scans | Behaviour at 200 scans | Behaviour over limit | +| --------------- | ----------- | ---------------------- | ------------------------------------ | +| **Anonymous** | {{ quota_anon }} | Reminder banner | CLI slows \~10 % | +| **Token (JWT)** | {{ quota_token }} | Reminder banner | Throttle continues, **never blocks** | + +*Request a free JWT:* send a blank e‑mail to +`token@stella-ops.org` – the bot replies with a signed token that you +store as `STELLA_JWT` in **`.env`**. + +--- + +## 5 · Updating the bundle ⤴️ + +1. Download the newer tarball & signature. +2. Repeat the **verify‑blob** step. +3. Run `stella ouk import ` – only the delta applies; average + upgrade time is **< 3 s**. + +--- + +## 6 · Road‑map highlights for Sovereign 🌐 + +| Release | Planned feature | +| ---------------------- | ---------------------------------------- | +| **v0.1 α (late 2025)** | Manual OUK import • Zastava beta | +| **v0.3 β (Q2 2026)** | Auto‑apply delta patch • nightly re‑scan | +| **v0.4 RC (Q3 2026)** | LDAP/AD SSO • registry scanner GA | +| **v1.0 GA (Q4 2026)** | Custom TLS/crypto adaptors (**incl. SM2**)—enabled where law or security requires it | + +Full details live in the public [Road‑map](../roadmap/README.md). + +--- + +## 7 · Troubleshooting 🩹 + +| Symptom | Fix | +| -------------------------------------------- | ------------------------------------------------------- | +| `cosign: signature mismatch` | File corrupted ‑ re‑download both tarball & `.sig` | +| `ouk import: no space left` | Ensure **8 GiB** free in `/var/lib/docker` | +| Import succeeds but scans still hit Internet | Confirm `STELLA_AIRGAP=true` in `.env` (v0.1‑α setting) | + +--- + +## 8 · FAQ — abbreviated ❓ + +
+Does the JWT token work offline? + +Yes. Signature validation happens locally; no outbound call is made. + +
+ +
+Can I mirror the bundle internally? + +Absolutely. Host the tarball on an intranet HTTP/S server or an object +store; signatures remain valid. + +
+ +
+Is there a torrent alternative? + +Planned for the β releases – follow the +[community chat](https://matrix.to/#/#stellaops:libera.chat) for ETA. + +
+ +--- + +### Licence & provenance 📜 + +The Offline Update Kit is part of Stella Ops and therefore +**AGPL‑3.0‑or‑later**. All components inherit the same licence. + +```bash +cosign verify-blob \ + --key https://stella-ops.org/keys/cosign.pub \ + --signature stella-ops-offline-usage-kit-v0.1a.tar.gz.sig \ + stella-ops-offline-usage-kit-v0.1a.tar.gz +``` + +— **Happy air‑gap scanning!** +© 2025‑2026 Stella Ops diff --git a/docs/10_PLUGIN_SDK_GUIDE.md b/docs/10_PLUGIN_SDK_GUIDE.md new file mode 100755 index 00000000..0956c046 --- /dev/null +++ b/docs/10_PLUGIN_SDK_GUIDE.md @@ -0,0 +1,194 @@ +# 10 · Plug‑in SDK Guide — **Stella Ops** +*(v 1.5 — 11 Jul 2025 · template install, no reload, IoC)* + +--- + +## 0 Audience & Scope +Guidance for developers who extend Stella Ops with schedule jobs, scanner adapters, TLS providers, notification channels, etc. Everything here is OSS; commercial variants simply ship additional signed plug‑ins. + +--- + +## 1 Prerequisites + +| Tool | Min Version | +| ----------------------- | ----------------------------------------------------------------- | +| .NET SDK | {{ dotnet }} | +| **StellaOps templates** | install once via `bash dotnet new install StellaOps.Templates::*` | +| **Cosign** | 2.3 + — used to sign DLLs | +| xUnit | 2.6 | +| Docker CLI | only if your plug‑in shells out to containers | + +--- + +## 2 Repository & Build Output + +Every plug‑in is hosted in **`git.stella‑ops.org`**. +At publish time it must copy its signed artefacts to: + +~~~text +src/backend/Stella.Ops.Plugin.Binaries// + ├── MyPlugin.dll + └── MyPlugin.dll.sig +~~~ + +The back‑end scans this folder on start‑up, verifies the **Cosign** signature, confirms the `[StellaPluginVersion]` gate, then loads the DLL inside an **isolated AssemblyLoadContext** to avoid dependency clashes + +--- + +## 3 Project Scaffold + +Generate with the installed template: + +~~~bash +dotnet new stellaops-plugin-schedule \ + -n MyPlugin.Schedule \ + --output src +~~~ + +Result: + +~~~text +src/ + ├─ MyPlugin.Schedule/ + │ ├─ MyJob.cs + │ └─ MyPlugin.Schedule.csproj + └─ tests/ + └─ MyPlugin.Schedule.Tests/ +~~~ + +--- + +## 4 MSBuild Wiring + +Add this to **`MyPlugin.Schedule.csproj`** so the signed DLL + `.sig` land in the canonical plug‑in folder: + +~~~xml + + $(SolutionDir)src/backend/Stella.Ops.Plugin.Binaries/$(MSBuildProjectName) + + + + + + + + + + + +~~~ + +--- + +## 5 Dependency‑Injection Entry‑point + +Back‑end auto‑discovers the static method below: + +~~~csharp +namespace StellaOps.DependencyInjection; + +public static class IoCConfigurator +{ + public static IServiceCollection Configure(this IServiceCollection services, + IConfiguration cfg) + { + services.AddSingleton(); // schedule job + services.Configure(cfg.GetSection("Plugins:MyPlugin")); + return services; + } +} +~~~ + +--- + +## 6 Schedule Plug‑ins + +### 6.1 Minimal Job + +~~~csharp +using StellaOps.Scheduling; // contract + +[StellaPluginVersion("2.0.0")] +public sealed class MyJob : IJob +{ + public async Task ExecuteAsync(CancellationToken ct) + { + Console.WriteLine("Hello from plug‑in!"); + await Task.Delay(500, ct); + } +} +~~~ + +### 6.2 Cron Registration + +```csharp +services.AddCronJob("0 15 * * *"); // everyday +``` + +15:00 +Cron syntax follows Hangfire rules  + +## 7 Scanner Adapters + +Implement IScannerRunner. +Register inside Configure: +```csharp +services.AddScanner("alt"); // backend +``` + +selects by --engine alt +If the engine needs a side‑car container, include a Dockerfile in your repo and document resource expectations. +## 8 Packaging & Signing + +```bash +dotnet publish -c Release -p:PublishSingleFile=true -o out +cosign sign --key $COSIGN_KEY out/MyPlugin.Schedule.dll # sign binary only +sha256sum out/MyPlugin.Schedule.dll > out/.sha256 # optional checksum +zip MyPlugin.zip out/* README.md +``` + +Unsigned DLLs are refused when StellaOps:Security:DisableUnsigned=false. + +## 9 Deployment + +```bash +docker cp MyPlugin.zip :/opt/plugins/ && docker restart +``` + +Check /health – "plugins":["MyPlugin.Schedule@2.0.0"]. +(Hot‑reload was removed to keep the core process simple and memory‑safe.) + +## 10 Configuration Patterns + +| Need | Pattern | +| ------------ | --------------------------------------------------------- | +| Settings | Plugins:MyPlugin:* in appsettings.json. | +| Secrets | Redis secure:: (encrypted per TLS provider). | +| Dynamic cron | Implement ICronConfigurable; UI exposes editor. | + +## 11 Testing & CI + +| Layer | Tool | Gate | +| ----------- | -------------------------- | ------------------- | +| Unit | xUnit + Moq | ≥ 50 % lines | +| Integration | Testcontainers ‑ run in CI | Job completes < 5 s | +| Style | dotnet | format 0 warnings | + +Use the pre‑baked workflow in StellaOps.Templates as starting point. + +## 12 Publishing to the Community Marketplace + +Tag Git release plugin‑vX.Y.Z and attach the signed ZIP. +Submit a PR to stellaops/community-plugins.json with metadata & git URL. +On merge, the plug‑in shows up in the UI Marketplace. + +## 13 Common Pitfalls + +| Symptom | Root cause | Fix | +| ------------------- | -------------------------- | ------------------------------------------- | +| NotDetected | .sig missing | cosign sign … | +| VersionGateMismatch | Backend 2.1 vs plug‑in 2.0 | Re‑compile / bump attribute | +| FileLoadException | Duplicate | StellaOps.Common Ensure PrivateAssets="all" | +| Redis | timeouts Large writes | Batch or use Mongo | \ No newline at end of file diff --git a/docs/11_DATA_SCHEMAS.md b/docs/11_DATA_SCHEMAS.md new file mode 100755 index 00000000..b68ac7ec --- /dev/null +++ b/docs/11_DATA_SCHEMAS.md @@ -0,0 +1,196 @@ +# Data Schemas & Persistence Contracts + +*Audience* – backend developers, plug‑in authors, DB admins. +*Scope* – describes **Redis**, **MongoDB** (optional), and on‑disk blob shapes that power Stella Ops. + +--- + +## 0 Document Conventions + +* **CamelCase** for JSON. +* All timestamps are **RFC 3339 / ISO 8601** with `Z` (UTC). +* `⭑` = planned but *not* shipped yet (kept on Feature Matrix “To Do”). + +--- + +## 1 SBOM Wrapper Envelope + +Every SBOM blob (regardless of format) is stored on disk or in object storage with a *sidecar* JSON file that indexes it for the scanners. + +#### 1.1 JSON Shape + +```jsonc +{ + "id": "sha256:417f…", // digest of the SBOM *file* itself + "imageDigest": "sha256:e2b9…", // digest of the original container image + "created": "2025-07-14T07:02:13Z", + "format": "trivy-json-v2", // NEW enum: trivy-json-v2 | spdx-json | cyclonedx-json + "layers": [ + "sha256:d38b…", // layer digests (ordered) + "sha256:af45…" + ], + "partial": false, // true => delta SBOM (only some layers) + "provenanceId": "prov_0291" // ⭑ link to SLSA attestation (Q1‑2026) +} +``` + +*`format`* **NEW** – added to support **multiple SBOM formats**. +*`partial`* **NEW** – true when generated via the **delta SBOM** flow (§1.3). + +#### 1.2 File‑system Layout + +``` +blobs/ + ├─ 417f… # digest prefix + │   ├─ sbom.json # payload (any format) + │   └─ sbom.meta.json # wrapper (shape above) +``` + +> **Note** – blob storage can point at S3, MinIO, or plain disk; driver plug‑ins adapt. + +#### 1.3 Delta SBOM Extension + +When `partial: true`, *only* the missing layers have been scanned. +Merging logic inside `scanning` module stitches new data onto the cached full SBOM in Redis. + +--- + +## 2 Redis Keyspace + +| Key pattern | Type | TTL | Purpose | +|-------------------------------------|---------|------|--------------------------------------------------| +| `scan:<digest>` | string | ∞ | Last scan JSON result (as returned by `/scan`) | +| `layers:<digest>` | set | 90d | Layers already possessing SBOMs (delta cache) | +| `policy:active` | string | ∞ | YAML **or** Rego ruleset | +| `quota:<token>` | string | *until next UTC midnight* | Per‑token scan counter for Free tier ({{ quota_token }} scans). | +| `policy:history` | list | ∞ | Change audit IDs (see Mongo) | +| `feed:nvd:json` | string | 24h | Normalised feed snapshot | +| `locator:<imageDigest>` | string | 30d | Maps image digest → sbomBlobId | +| `metrics:…` | various | — | Prom / OTLP runtime metrics | + +> **Delta SBOM** uses `layers:*` to skip work in <20 ms. +> **Quota enforcement** increments `quota:` atomically; when {{ quota_token }} the API returns **429**. + +--- + +## 3 MongoDB Collections (Optional) + +Only enabled when `MONGO_URI` is supplied (for long‑term audit). + +| Collection | Shape (summary) | Indexes | +|--------------------|------------------------------------------------------------|-------------------------------------| +| `sbom_history` | Wrapper JSON + `replaceTs` on overwrite | `{imageDigest}` `{created}` | +| `policy_versions` | `{_id, yaml, rego, authorId, created}` | `{created}` | +| `attestations` ⭑ | SLSA provenance doc + Rekor log pointer | `{imageDigest}` | +| `audit_log` | Fully rendered RFC 5424 entries (UI & CLI actions) | `{userId}` `{ts}` | + +Schema detail for **policy_versions**: + +```jsonc +{ + "_id": "6619e90b8c5e1f76", + "yaml": "version: 1.0\nrules:\n - …", + "rego": null, // filled when Rego uploaded + "authorId": "u_1021", + "created": "2025-07-14T08:15:04Z", + "comment": "Imported via API" +} +``` + +--- + +## 4 Policy Schema (YAML v1.0) + +Minimal viable grammar (subset of OSV‑SCHEMA ideas). + +```yaml +version: "1.0" +rules: + - name: Block Critical + severity: [Critical] + action: block + - name: Ignore Low Dev + severity: [Low, None] + environments: [dev, staging] + action: ignore + expires: "2026-01-01" + - name: Escalate RegionalFeed High + sources: [NVD, CNNVD, CNVD, ENISA, JVN, BDU] + severity: [High, Critical] + action: escalate +``` + +Validation is performed by `policy:mapping.yaml` JSON‑Schema embedded in backend. + +### 4.1 Rego Variant (Advanced – TODO) + +*Accepted but stored as‑is in `rego` field.* +Evaluated via internal **OPA** side‑car once feature graduates from TODO list. + +--- + +## 5 SLSA Attestation Schema ⭑ + +Planned for Q1‑2026 (kept here for early plug‑in authors). + +```jsonc +{ + "id": "prov_0291", + "imageDigest": "sha256:e2b9…", + "buildType": "https://slsa.dev/container/v1", + "builder": { + "id": "https://git.stella-ops.ru/ci/stella-runner@sha256:f7b7…" + }, + "metadata": { + "invocation": { + "parameters": {"GIT_SHA": "f6a1…"}, + "buildStart": "2025-07-14T06:59:17Z", + "buildEnd": "2025-07-14T07:01:22Z" + }, + "completeness": {"parameters": true} + }, + "materials": [ + {"uri": "git+https://git…", "digest": {"sha1": "f6a1…"}} + ], + "rekorLogIndex": 99817 // entry in local Rekor mirror +} +``` + +--- + +## 6 Validator Contracts + +* For SBOM wrapper – `ISbomValidator` (DLL plug‑in) must return *typed* error list. +* For YAML policies – JSON‑Schema at `/schemas/policy‑v1.json`. +* For Rego – OPA `opa eval --fail-defined` under the hood. +* For **Free‑tier quotas** – `IQuotaService` integration tests ensure `quota:` resets at UTC midnight and produces correct `Retry‑After` headers. + +--- + +## 7 Migration Notes + +1. **Add `format` column** to existing SBOM wrappers; default to `trivy-json-v2`. +2. **Populate `layers` & `partial`** via backfill script (ship with `stellopsctl migrate` wizard). +3. Policy YAML previously stored in Redis → copy to Mongo if persistence enabled. +4. Prepare `attestations` collection (empty) – safe to create in advance. + +--- + +## 8 Open Questions / Future Work + +* How to de‑duplicate *identical* Rego policies differing only in whitespace? +* Embed *GOST 34.11‑2018* digests when users enable Russian crypto suite? +* Should enterprise tiers share the same Redis quota keys or switch to JWT claim `tier != Free` bypass? +* Evaluate sliding‑window quota instead of strict daily reset. +* Consider rate‑limit for `/layers/missing` to avoid brute‑force enumeration. + +--- + +## 9 Change Log + +| Date | Note | +|------------|--------------------------------------------------------------------------------| +| 2025‑07‑14 | **Added:** `format`, `partial`, delta cache keys, YAML policy schema v1.0. | +| 2025‑07‑12 | **Initial public draft** – SBOM wrapper, Redis keyspace, audit collections. | + +--- diff --git a/docs/11_GOVERNANCE.md b/docs/11_GOVERNANCE.md new file mode 100755 index 00000000..2b835033 --- /dev/null +++ b/docs/11_GOVERNANCE.md @@ -0,0 +1,93 @@ +# Stella Ops Project Governance +*Lazy Consensus • Maintainer Charter • Transparent Veto* + +> **Scope** – applies to **all** repositories under +> `https://git.stella-ops.org/stella-ops/*` unless a sub‑project overrides it +> with its own charter approved by the Core Maintainers. + +--- + +## 1 · Decision‑making workflow 🗳️ + +| Stage | Default vote | Timer | +|-------|--------------|-------| +| **Docs / non‑code PR** | `+1` | **48 h** | +| **Code / tests PR** | `+1` | **7 × 24 h** | +| **Security‑sensitive / breaking API** | `+1` + explicit **`security‑LGTM`** | **7 × 24 h** | + +**Lazy‑consensus** – silence = approval once the timer elapses. + +* **Veto `‑1`** must include a concrete concern **and** a path to resolution. +* After 3 unresolved vetoes the PR escalates to a **Maintainer Summit** call. + +--- + +## 2 · Maintainer approval thresholds 👥 + +| Change class | Approvals required | Example | +|--------------|-------------------|---------| +| **Trivial** | 0 | Typos, comment fixes | +| **Non‑trivial** | **2 Maintainers** | New API endpoint, feature flag | +| **Security / breaking** | Lazy‑consensus **+ `security‑LGTM`** | JWT validation, crypto swap | + +Approval is recorded via Git forge review or a signed commit trailer +`Signed-off-by: `. + +--- + +## 3 · Becoming (and staying) a Maintainer 🌱 + +1. **3 + months** of consistent, high‑quality contributions. +2. **Nomination** by an existing Maintainer via issue. +3. **7‑day vote** – needs ≥ **⅔ majority** “`+1`”. +4. Sign `MAINTAINER_AGREEMENT.md` and enable **2FA**. +5. Inactivity > 6 months → automatic emeritus status (can be re‑activated). + +--- + +## 4 · Release authority & provenance 🔏 + +* Every tag is **co‑signed by at least one Security Maintainer**. +* CI emits a **signed SPDX SBOM** + **Cosign provenance**. +* Release cadence is fixed – see [public Road‑map](../roadmap/README.md). +* Security fixes may create out‑of‑band `x.y.z‑hotfix` tags. + +--- + +## 5 · Escalation lanes 🚦 + +| Situation | Escalation | +|-----------|------------| +| Technical deadlock | **Maintainer Summit** (recorded & published) | +| Security bug | Follow [Security Policy](../security/01_SECURITY_POLICY.md) | +| Code of Conduct violation | See `12_CODE_OF_CONDUCT.md` escalation ladder | + +--- + +## 6 · Contribution etiquette 🤝 + +* Draft PRs early – CI linting & tests help you iterate. +* “There are no stupid questions” – ask in **Matrix #dev**. +* Keep commit messages in **imperative mood** (`Fix typo`, `Add SBOM cache`). +* Run the `pre‑commit` hook locally before pushing. + +--- + +## 7 · Licence reminder 📜 + +Stella Ops is **AGPL‑3.0‑or‑later**. By contributing you agree that your +patches are released under the same licence. + +--- + +### Appendix A – Maintainer list 📇 + +*(Generated via `scripts/gen-maintainers.sh` – edit the YAML, **not** this +section directly.)* + +| Handle | Area | Since | +|--------|------|-------| +| `@alice` | Core scanner • Security | 2025‑04 | +| `@bob` | UI • Docs | 2025‑06 | + +--- \ No newline at end of file diff --git a/docs/12_CODE_OF_CONDUCT.md b/docs/12_CODE_OF_CONDUCT.md new file mode 100755 index 00000000..236b4ab8 --- /dev/null +++ b/docs/12_CODE_OF_CONDUCT.md @@ -0,0 +1,88 @@ +# Stella Ops Code of Conduct +*Contributor Covenant v2.1 + project‑specific escalation paths* + +> We pledge to make participation in the Stella Ops community a +> harassment‑free experience for everyone, regardless of age, body size, +> disability, ethnicity, sex characteristics, gender identity and expression, +> level of experience, education, socio‑economic status, nationality, +> personal appearance, race, religion, or sexual identity and orientation. + +--- + +## 0 · Our standard + +This project adopts the +[**Contributor Covenant v2.1**](https://www.contributor-covenant.org/version/2/1/code_of_conduct/) +with the additions and clarifications listed below. +If anything here conflicts with the upstream covenant, *our additions win*. + +--- + +## 1 · Scope + +| Applies to | Examples | +|------------|----------| +| **All official spaces** | Repos under `git.stella-ops.org/stella-ops.org/*`, Matrix rooms (`#stellaops:*`), issue trackers, pull‑request reviews, community calls, and any event officially sponsored by Stella Ops | +| **Unofficial spaces that impact the project** | Public social‑media posts that target or harass community members, coordinated harassment campaigns, doxxing, etc. | + +--- + +## 2 · Reporting a violation ☎️ + +| Channel | When to use | +|---------|-------------| +| `conduct@stella-ops.org` (PGP key [`keys/#pgp`](../keys/#pgp)) | **Primary, confidential** – anything from micro‑aggressions to serious harassment | +| Matrix `/msg @coc-bot:libera.chat` | Quick, in‑chat nudge for minor issues | +| Public issue with label `coc` | Transparency preferred and **you feel safe** doing so | + +We aim to acknowledge **within 48 hours** (business days, UTC). + +--- + +## 3 · Incident handlers 🛡️ + +| Name | Role | Alt‑contact | +|------|------|-------------| +| Alice Doe (`@alice`) | Core Maintainer • Security WG | `+1‑555‑0123` | +| Bob Ng (`@bob`) | UI Maintainer • Community lead | `+1‑555‑0456` | + +If **any** handler is the subject of a complaint, skip them and contact another +handler directly or email `conduct@stella-ops.org` only. + +--- + +## 4 · Enforcement ladder ⚖️ + +1. **Private coaches / mediation** – first attempt to resolve misunderstandings. +2. **Warning** – written, includes corrective actions & cooling‑off period. +3. **Temporary exclusion** – mute (chat), read‑only (repo) for *N* days. +4. **Permanent ban** – removal from all official spaces + revocation of roles. + +All decisions are documented **privately** (for confidentiality) but a summary +is published quarterly in the “Community Health” report. + +--- + +## 5 · Appeals 🔄 + +A sanctioned individual may appeal **once** by emailing +`appeals@stella-ops.org` within **14 days** of the decision. +Appeals are reviewed by **three maintainers not involved in the original case** +and resolved within 30 days. + +--- + +## 6 · No‑retaliation policy 🛑 + +Retaliation against reporters **will not be tolerated** and results in +immediate progression to **Step 4** of the enforcement ladder. + +--- + +## 7 · Attribution & licence 📜 + +* Text adapted from Contributor Covenant v2.1 – + Copyright © 2014‑2024 Contributor Covenant Contributors + Licensed under [CC BY 4.0](https://creativecommons.org/licenses/by/4.0/). + +--- diff --git a/docs/12_PERFORMANCE_WORKBOOK.md b/docs/12_PERFORMANCE_WORKBOOK.md new file mode 100755 index 00000000..192fecc3 --- /dev/null +++ b/docs/12_PERFORMANCE_WORKBOOK.md @@ -0,0 +1,167 @@ +# 12 - Performance Workbook + +*Purpose* – define **repeatable, data‑driven** benchmarks that guard Stella Ops’ core pledge: +> *“P95 vulnerability feedback in ≤ 5 seconds.”* + +--- + +## 0 Benchmark Scope + +| Area | Included | Excluded | +|------------------|----------------------------------|---------------------------| +| SBOM‑first scan | Trivy engine w/ warmed DB | Full image unpack ≥ 300 MB | +| Delta SBOM ⭑ | Missing‑layer lookup & merge | Multi‑arch images | +| Policy eval ⭑ | YAML → JSON → rule match | Rego (until GA) | +| Feed merge | NVD JSON 2023–2025 | GHSA GraphQL (plugin) | +| Quota wait‑path | 5 s soft‑wait, 60 s hard‑wait behaviour | Paid tiers (unlimited) | +| API latency | REST `/scan`, `/layers/missing` | UI SPA calls | + +⭑ = new in July 2025. + +--- + +## 1 Hardware Baseline (Reference Rig) + +| Element | Spec | +|-------------|------------------------------------| +| CPU | 8 vCPU (Intel Ice‑Lake equiv.) | +| Memory | 16 GiB | +| Disk | NVMe SSD, 3 GB/s R/W | +| Network | 1 Gbit virt. switch | +| Container | Docker 25.0 + overlay2 | +| OS | Ubuntu 22.04 LTS (kernel 6.8) | + +*All P95 targets assume a **single‑node** deployment on this rig unless stated.* + +--- + +## 2 Phase Targets & Gates + +| Phase (ID) | Target P95 | Gate (CI) | Rationale | +|-----------------------|-----------:|-----------|----------------------------------------| +| **SBOM_FIRST** | ≤ 5 s | `hard` | Core UX promise. | +| **IMAGE_UNPACK** | ≤ 10 s | `soft` | Fallback path for legacy flows. | +| **DELTA_SBOM** ⭑ | ≤ 1 s | `hard` | Needed to stay sub‑5 s for big bases. | +| **POLICY_EVAL** ⭑ | ≤ 50 ms | `hard` | Keeps gate latency invisible to users. | +| **QUOTA_WAIT** ⭑ | *soft* ≤ 5 s
*hard* ≤ 60 s | `hard` | Ensures graceful Free‑tier throttling. | +| **SCHED_RESCAN** | ≤ 30 s | `soft` | Nightly batch – not user‑facing. | +| **FEED_MERGE** | ≤ 60 s | `soft` | Off‑peak cron @ 01:00. | +| **API_P95** | ≤ 200 ms | `hard` | UI snappiness. | + +*Gate* legend — `hard`: break CI if regression > 3 × target, +`soft`: raise warning & issue ticket. + +--- + +## 3 Test Harness + +* **Runner** – `perf/run.sh`, accepts `--phase` and `--samples`. +* **Metrics** – Prometheus + `jq` extracts; aggregated via `scripts/aggregate.ts`. +* **CI** – GitLab CI job *benchmark* publishes JSON to `bench‑artifacts/`. +* **Visualisation** – Grafana dashboard *Stella‑Perf* (provisioned JSON). + +> **Note** – harness mounts `/var/cache/trivy` tmpfs to avoid disk noise. + +--- + +## 4 Current Results (July 2025) + +| Phase | Samples | Mean (s) | P95 (s) | Target OK? | +|---------------|--------:|---------:|--------:|-----------:| +| SBOM_FIRST | 100 | 3.7 | 4.9 | ✅ | +| IMAGE_UNPACK | 50 | 6.4 | 9.2 | ✅ | +| **DELTA_SBOM**| 100 | 0.46 | 0.83 | ✅ | +| **POLICY_EVAL** | 1 000 | 0.021 | 0.041 | ✅ | +| **QUOTA_WAIT** | 80 | 4.0* | 4.9* | ✅ | +| SCHED_RESCAN | 10 | 18.3 | 24.9 | ✅ | +| FEED_MERGE | 3 | 38.1 | 41.0 | ✅ | +| API_P95 | 20 000 | 0.087 | 0.143 | ✅ | + +*Data files:* `bench-artifacts/2025‑07‑14/phase‑stats.json`. + +--- + +## 5 Δ‑SBOM Micro‑Benchmark Detail + +### 5.1 Scenario + +1. Base image `python:3.12-slim` already scanned (all layers cached). +2. Application layer (`COPY . /app`) triggers new digest. +3. `Stella CLI` lists **7** layers, backend replies *6 hit*, *1 miss*. +4. Builder scans **only 1 layer** (~9 MiB, 217 files) & uploads delta. + +### 5.2 Key Timings + +| Step | Time (ms) | +|---------------------|----------:| +| `/layers/missing` | 13 | +| Trivy single layer | 655 | +| Upload delta blob | 88 | +| Backend merge + CVE | 74 | +| **Total wall‑time** | **830 ms** | + +--- + +## 6 Quota Wait‑Path Benchmark Detail + +### 6.1 Scenario + +1. Free‑tier token reaches **scan #200** – dashboard shows yellow banner. + +### 6.2 Key Timings + +| Step | Time (ms) | +|------------------------------------|----------:| +| `/quota/check` Redis LUA INCR | 0.8 | +| Soft wait sleep (server) | 5 000 | +| Hard wait sleep (server) | 60 000 | +| End‑to‑end wall‑time (soft‑hit) | 5 003 | +| End‑to‑end wall‑time (hard‑hit) | 60 004 | + +--- +## 7 Policy Eval Bench + +### 7.1 Setup + +* Policy YAML: **28** rules, mix severity & package conditions. +* Input: scan result JSON with **1 026** findings. +* Evaluator: custom rules engine (Go structs → map look‑ups). + +### 7.2 Latency Histogram + +``` +0‑10 ms ▇▇▇▇▇▇▇▇▇▇ 38 % +10‑20 ms ▇▇▇▇▇▇▇▇▇▇ 42 % +20‑40 ms ▇▇▇▇▇▇ 17 % +40‑50 ms ▇ 3 % +``` + +P99 = 48 ms. Meets 50 ms gate. + +--- + +## 8 Trend Snapshot + +![Perf trend spark‑line placeholder](perf‑trend.svg) + +_Plot generated weekly by `scripts/update‑trend.py`; shows last 12 weeks P95 per phase._ + +--- + +## 9 Action Items + +1. **Image Unpack** – Evaluate zstd for layer decompress; aim to shave 1 s. +2. **Feed Merge** – Parallelise regional XML feed parse (plugin) once stable. +3. **Rego Support** – Prototype OPA side‑car; target ≤ 100 ms eval. +4. **Concurrency** – Stress‑test 100 rps on 4‑node Redis cluster (Q4‑2025). + +--- + +## 10 Change Log + +| Date | Note | +|------------|-------------------------------------------------------------------------| +| 2025‑07‑14 | Added Δ‑SBOM & Policy Eval phases; updated targets & current results. | +| 2025‑07‑12 | First public workbook (SBOM‑first, image‑unpack, feed merge). | + +--- diff --git a/docs/13_RELEASE_ENGINEERING_PLAYBOOK.md b/docs/13_RELEASE_ENGINEERING_PLAYBOOK.md new file mode 100755 index 00000000..2586f655 --- /dev/null +++ b/docs/13_RELEASE_ENGINEERING_PLAYBOOK.md @@ -0,0 +1,209 @@ +# 13 · Release Engineering Playbook — Stella Ops + + +A concise, automation‑first guide describing **how source code on `main` becomes a verifiably signed, air‑gap‑friendly release**. +It is opinionated for offline use‑cases and supply‑chain security (SLSA ≥ level 2 today, aiming for level 3). + +--- + +## 0 Release Philosophy + +* **Fast but fearless** – every commit on `main` must be releasable; broken builds break the build, not the team. +* **Reproducible** – anyone can rebuild byte‑identical artefacts with a single `make release` offline. +* **Secure by default** – every artefact ships with a SBOM, Cosign signature and (future) Rekor log entry. +* **Offline‑first** – all dependencies are vendored or mirrored into the internal registry; no Internet required at runtime. + +--- + +## 1 Versioning & Branching + +| Branch | Purpose | Auto‑publish? | +| ------------- | ------------------------------ | --------------------------------------- | +| `main` | Always‑green development trunk | `nightly-*` images | +| `release/X.Y` | Stabilise a minor line | `stella:X.Y-rcN` | +| Tags | `X.Y.Z` = SemVer | `stella:X.Y.Z`, OUK tarball, Helm chart | + +* **SemVer** – MAJOR for breaking API/CLI changes, MINOR for features, PATCH for fixes. +* Release tags are **signed** (`git tag -s`) with the Stella Ops GPG key (`0x90C4…`). + +--- + +## 2 CI/CD Overview (GitLab CI + GitLab Runner) + +```mermaid +graph LR + A[push / MR] --> Lint + Lint --> Unit + Unit --> Build + Build --> Test-Container + Test-Container --> SBOM + SBOM --> Sign + Sign --> Publish + Publish --> E2E + Publish --> Notify +``` + +### Pipeline Stages + +| Stage | Key tasks | +| ------------------ | ------------------------------------------------------------------------------------------------ | +| **Lint** | ESLint, golangci‑lint, hadolint, markdown‑lint. | +| **Unit** | `dotnet test`, `go test`, Jest UI tests. | +| **Quota unit‑tests 🏷** | Validate QuotaService logic: reset at UTC, 5 s vs 60 s waits, header correctness. | +| **Build** | Multi‑arch container build (`linux/amd64`, `linux/arm64`) using **BuildKit** + `--provenance` 📌. | +| **Test‑Container** | Spin up compose file, run smoke APIs. | +| **SBOM** 📌 | Invoke **StellaOps.SBOMBuilder** to generate SPDX JSON + attach `.sbom` label to image. | +| **Sign** | Sign image with **Cosign** (`cosign sign --key cosign.key`). | +| **Publish** | Push to `registry.git.stella-ops.org`. | +| **E2E** | Kind‑based Kubernetes test incl. Zastava DaemonSet; verify sub‑5 s scan SLA. | +| **Notify** | Report to Mattermost & GitLab Slack app. | +| **OfflineToken** | Call `JwtIssuer.Generate(exp=30d)` → store `client.jwt` artefact → attach to OUK build context | + +*All stages run in parallel where possible; max wall‑time < 15 min.* + +--- + +## 3 Container Image Strategy + +| Image | Registry Tag | Contents | +| ------------------------------ | --------------------------- | ---------------------------------------------------------------------- | +| **backend** | `stella/backend:{ver}` | ASP.NET API, plugin loader. | +| **ui** | `stella/ui:{ver}` | Pre‑built Angular SPA. | +| **runner-trivy** | `stella/runner-trivy:{ver}` | Trivy CLI + SPDX/CycloneDX 🛠. | +| **runner-grype** | `stella/runner-grype:{ver}` | Optional plug‑in scanner. | +| **🏷️ StellaOps.Registry** 📌 | `stella/registry:{ver}` | Scratch image embedding Docker Registry v2 + Cosign policy controller. | +| **🏷️ StellaOps.MutePolicies** 📌 | `stella/policies:{ver}` | Sidecar serving policy bundles. | +| **🏷️ StellaOps.Attestor** 📌 | `stella/attestor:{ver}` | SLSA provenance & Rekor signer (future). | + +*Images are **`--label org.opencontainers.image.source=git.stella-ops.ru`** and include SBOMs generated at build time.* + +--- + +## 4 📌 Offline Update Kit (OUK) Build & Distribution + +**Purpose** – deliver updated CVE feeds & Trivy DB to air‑gapped clusters. + +### 4.1 CLI Tool + +*Go binary `ouk` lives in `tools/ouk/`.* + +```sh +ouk fetch \ + --nvd --osv \ + --trivy-db --date $(date -I) \ + --output ouk-$(date +%Y%m%d).tar.gz \ + --sign cosign.key +``` + +### 4.2 Pipeline Hook + +* Runs on **first Friday** each month (cron). +* Generates tarball, signs it, uploads to **GitLab Release asset**. +* SHA‑256 + signature published alongside. + +### 4.3 Activation Flow (runtime) + +1. Admin uploads `.tar.gz` via **UI → Settings → Offline Updates (OUK)**. +2. Backend verifies Cosign signature & digest. +3. Files extracted into `var/lib/stella/db`. +4. Redis caches invalidated; Dashboard “Feed Age” ticks green. +5. Audit event `ouk_update` stored. + +### 4.4 Token Detail + +client.jwt placed under /root/ inside the tarball. +CI job fails if token expiry < 29 days (guard against stale caches). + +--- + +## 5 Artifact Signing & Transparency + +| Artefact | Signer | Tool | +| ------------ | --------------- | --------------------- | +| Git tags | GPG (`0x90C4…`) | `git tag -s` | +| Containers | Cosign key pair | `cosign sign` | +| Helm Charts | prov file | `helm package --sign` | +| OUK tarballs | Cosign | `cosign sign-blob` | + +**Rekor** integration is **TODO** – once the internal Rekor mirror is online (`StellaOpsAttestor`) a post‑publish job will submit transparency log entries. + +--- + +## 6 Release Checklist + +1. CI pipeline green. +2. Bump `VERSION` file. +3. Tag `git tag -s X.Y.Z -m "Release X.Y.Z"` & push. +4. GitLab CI auto‑publishes images & charts. +5. Draft GitLab **Release Notes** using `tools/release-notes-gen`. +6. Verify SBOM attachment with `stella sbom verify stella/backend:X.Y.Z`. +7. Smoke‑test OUK tarball in offline lab. +8. Announce in `#stella-release` Mattermost channel. + +--- + +## 7 Hot‑fix Procedure + +* Branch from latest tag → `hotfix/X.Y.Z+1-hf1`. +* Apply minimal patch, add regression test. +* CI pipeline (with reduced stages) must pass. +* Tag `X.Y.Z+1`. +* Publish only container + Helm chart; OUK not rebuilt. +* Cherry‑pick back to `main`. + +--- + +## 8 Deprecation & End‑of‑Life Policy + +| Feature | Deprecation notice | Removal earliest | +| ------------------------ | ------------------ | ---------------- | +| Legacy CSV policy import | 2025‑10‑01 | 2026‑04‑01 | +| Docker v1 Registry auth | 2025‑12‑01 | 2026‑06‑01 | +| In‑image Trivy DB | 2025‑12‑15 | 2026‑03‑15 | + +*At least 6 months notice; removal requires major version bump.* + +--- + +## 9 📌 Non‑Commercial Usage Rules (English canonical) + +1. **Free for internal security assessments** (company or personal). +2. **SaaS resale / re‑hosting prohibited** without prior written consent (AGPL §13). +3. If you distribute a fork with UI or backend modifications **you must**: + * Publish the complete modified source code. + * Retain the original Stella Ops attribution in UI footer and CLI `--version`. +4. All third‑party dependencies remain under their respective licences (MIT, Apache‑2.0, ISC, BSD). +5. Deployments in state‑regulated or classified environments must obey**applicable local regulations** governing cryptography and software distribution. + +--- + +## 10 Best Practices Snapshot 📌 + +* **SBOM‑per‑image** → attach at build time; store as OCI artifact for supply‑chain introspection. +* **Provenance flag** (`--provenance=true`) in BuildKit fulfils SLSA 2 requirement. +* Use **multi‑arch, reproducible builds** (`SOURCE_DATE_EPOCH` pins timestamps). +* All pipelines enforce **Signed‑off‑by (DCO)**; CI fails if trailer missing. +* `cosign policy` ensures only images signed by the project key run in production. + +--- + +## 11 Contributing to Release Engineering + +* Fork & create MR to `infra/release-*`. +* All infra changes require green **`integration-e2e-offline`** job. +* Discuss larger infra migrations in `#sig-release` Mattermost; decisions recorded in `ADR/` folder. + +--- + +## 12 Change Log (high‑level) + +| Version | Date | Note | +| ------- | ---------- | --------------------------------------------------------------------------------------------------------------------------------------------------------- | +| v2.1 | 2025‑07‑15 | Added OUK build/publish pipeline, internal registry image (`StellaOps.Registry`), non‑commercial usage rules extraction, SBOM stage, BuildKit provenance. | +| v2.0 | 2025‑07‑12 | Initial open‑sourcing of Release Engineering guide. | +| v1.1 | 2025‑07‑09 | Fixed inner fencing; added retention policy | +| v1.0 | 2025‑07‑09 | Initial playbook | + +--- + +*(End of Release Engineering Playbook v1.1)* diff --git a/docs/13_SECURITY_POLICY.md b/docs/13_SECURITY_POLICY.md new file mode 100755 index 00000000..1956a68d --- /dev/null +++ b/docs/13_SECURITY_POLICY.md @@ -0,0 +1,101 @@ +# Stella Ops Security Policy & Responsible Disclosure +*Version 3 · 2025‑07‑15* + +--- + +## 0 · Supported versions 🗓️ + +| Release line | Status | Security fix window | +|--------------|--------|---------------------| +| **v0.1 α** (late 2025) | *Upcoming* | 90 days after GA of v0.2 | +| **v0.2 β** (Q1 2026) | *Planned* | 6 months after GA of v0.3 | +| **v0.3 β** (Q2 2026) | *Planned* | 6 months after GA of v0.4 | +| **v0.4 RC** (Q3 2026) | *Planned* | Until v1.0 GA | +| **v1.0 GA** (Q4 2026) | *Future LTS* | 24 months from release | + +Pre‑GA lines receive **critical** and **high**‑severity fixes only. + +--- + +## 1 · How to report a vulnerability 🔒 + +| Channel | PGP‑encrypted? | Target SLA | +|---------|---------------|-----------| +| `security@stella-ops.org` | **Yes** – PGP key: [`/keys/#pgp`](../keys/#pgp) | 72 h acknowledgement | +| Matrix DM → `@sec‑bot:libera.chat` | Optional | 72 h acknowledgement | +| Public issue with label `security` | No (for non‑confidential flaws) | 7 d acknowledgement | + +Please include: + +* Affected version(s) and environment +* Reproduction steps or PoC +* Impact assessment (data exposure, RCE, DoS, etc.) +* Preferred disclosure timeline / CVE request info + +--- + +## 2 · Our disclosure process 📜 + +1. **Triage** – confirm the issue, assess severity, assign CVSS v4 score. +2. **Patch development** – branch created in a private mirror; PoCs kept confidential. +3. **Pre‑notification** – downstream packagers & large adopters alerted **72 h** before release. +4. **Co‑ordinated release** – patched version + advisory (GHSA + CVE) + SBOM delta. +5. **Credits** – researchers listed in release notes (opt‑in). + +We aim for **30 days** from report to release for critical/high issues; medium/low may wait for the next scheduled release. + +--- + +## 3 · Existing safeguards ✅ + +| Layer | Control | +|-------|---------| +| **Release integrity** | `cosign` signatures + SPDX SBOM on every artefact | +| **Build pipeline** | Reproducible, fully declarative CI; SBOM diff verified in CI | +| **Runtime hardening** | Non‑root UID, distroless‑glibc base, SELinux/AppArmor profiles, cgroup CPU/RAM caps | +| **Access logs** | Retained **7 days**, then `sha256(ip)` hash | +| **Quota ledger** | Stores *token‑ID hash* only, no plain e‑mail/IP | +| **Air‑gap support** | Signed **Offline Update Kit** (OUK) validated before import | +| **Secure defaults** | TLS 1.3 (or stronger via plug‑in), HTTP Strict‑Transport‑Security, Content‑Security‑Policy | +| **SBOM re‑scan** | Nightly cron re‑checks previously “clean” images against fresh CVE feeds | + +--- + +## 4 · Cryptographic keys 🔑 + +| Purpose | Fingerprint | Where to fetch | +|---------|-------------|----------------| +| **PGP (sec‑team)** | `3A5C ​71F3 ​... ​7D9B` | [`/keys/#pgp`](../keys/#pgp) | +| **Cosign release key** | `AB12 ... EF90` | [`/keys/#cosign`](../keys/#cosign) | + +Verify all downloads (TLS 1.3 by default; 1.2 allowed only via a custom TLS provider such as GOST): + + +```bash +cosign verify \ + --key https://stella-ops.org/keys/cosign.pub \ + registry.stella-ops.org/stella-ops/stella-ops: +```` + +--- + +## 5 · Private‑feed mirrors 🌐 + +The **Feedser (vulnerability ingest/merge/export service)** provides signed JSON and Trivy DB snapshots that merge: + +* OSV + GHSA +* (optional) NVD 2.0, CNNVD, CNVD, ENISA, JVN and BDU regionals + +The snapshot ships in every Offline Update Kit and is validated with an in‑toto SLSA attestation at import time. + +--- + +## 6 · Hall of Thanks 🏆 + +We are grateful to the researchers who help keep Stella Ops safe: + +| Release | Researcher | Handle / Org | +| ------- | ------------------ | ------------ | +| *empty* | *(your name here)* | | + +--- diff --git a/docs/14_GLOSSARY_OF_TERMS.md b/docs/14_GLOSSARY_OF_TERMS.md new file mode 100755 index 00000000..300c0ade --- /dev/null +++ b/docs/14_GLOSSARY_OF_TERMS.md @@ -0,0 +1,112 @@ +# 14 · Glossary of Terms — Stella Ops + + +--- + +### 0 Purpose +A concise, single‑page **“what does that acronym actually mean?”** reference for +developers, DevOps engineers, IT managers and auditors who are new to the +Stella Ops documentation set. + +*If you meet a term in any Stella Ops doc that is **not** listed here, please +open a PR and append it alphabetically.* + +--- + +## A – C + +| Term | Short definition | Links / notes | +|------|------------------|---------------| +| **ADR** | *Architecture Decision Record* – lightweight Markdown file that captures one irreversible design decision. | ADR template lives at `/docs/adr/` | +| **AIRE** | *AI Risk Evaluator* – optional Plus/Pro plug‑in that suggests mute rules using an ONNX model. | Commercial feature | +| **Azure‑Pipelines** | CI/CD service in Microsoft Azure DevOps. | Recipe in Pipeline Library | +| **BDU** | Russian (FSTEC) national vulnerability database: *База данных уязвимостей*. | Merged with NVD by Feedser (vulnerability ingest/merge/export service) | +| **BuildKit** | Modern Docker build engine with caching and concurrency. | Needed for layer cache patterns | +| **CI** | *Continuous Integration* – automated build/test pipeline. | Stella integrates via CLI | +| **Cosign** | Open‑source Sigstore tool that signs & verifies container images **and files**. | Images & OUK tarballs | +| **CWV / CLS** | *Core Web Vitals* metric – Cumulative Layout Shift. | UI budget ≤ 0.1 | +| **CycloneDX** | Open SBOM (BOM) standard alternative to SPDX. | Planned report format plug‑in | + +--- + +## D – G + +| Term | Definition | Notes | +|------|------------|-------| +| **Digest (image)** | SHA‑256 hash uniquely identifying a container image or layer. | Pin digests for reproducible builds | +| **Docker‑in‑Docker (DinD)** | Running Docker daemon inside a CI container. | Used in GitHub / GitLab recipes | +| **DTO** | *Data Transfer Object* – C# record serialised to JSON. | Schemas in doc 11 | +| **Feedser** | Vulnerability ingest/merge/export service consolidating OVN, GHSA, NVD 2.0, CNNVD, CNVD, ENISA, JVN and BDU feeds into the canonical MongoDB store and export artifacts. | Cron default `0 1 * * *` | +| **FSTEC** | Russian regulator issuing SOBIT certificates. | Pro GA target | +| **Gitea** | Self‑hosted Git service – mirrors GitHub repo. | OSS hosting | +| **GOST TLS** | TLS cipher‑suites defined by Russian GOST R 34.10‑2012 / 34.11‑2012. | Provided by `OpenSslGost` or CryptoPro | +| **Grype** | Alternative OSS vulnerability scanner; can be hot‑loaded as plug‑in. | Scanner interface `IScannerRunner` | + +--- + +## H – L + +| Term | Definition | Notes | +|------|------------|-------| +| **Helm** | Kubernetes package manager (charts). | Beta chart under `/charts/core` | +| **Hot‑load** | Runtime discovery & loading of plug‑ins **without restart**. | Cosign‑signed DLLs | +| **Hyperfine** | CLI micro‑benchmark tool used in Performance Workbook. | Outputs CSV | +| **JWT** | *JSON Web Token* – bearer auth token issued by OpenIddict. | Scope `scanner`, `admin`, `ui` | +| **K3s / RKE2** | Lightweight Kubernetes distributions (Rancher). | Supported in K8s guide | +| **Kubernetes NetworkPolicy** | K8s resource controlling pod traffic. | Redis/Mongo isolation | + +--- + +## M – O + +| Term | Definition | Notes | +|------|------------|-------| +| **Mongo (optional)** | Document DB storing > 180 day history and audit logs. | Off by default in Core | +| **Mute rule** | JSON object that suppresses specific CVEs until expiry. | Schema `mute-rule‑1.json` | +| **NVD** | US‑based *National Vulnerability Database*. | Primary CVE source | +| **ONNX** | Portable neural‑network model format; used by AIRE. | Runs in‑process | +| **OpenIddict** | .NET library that implements OAuth2 / OIDC in Stella backend. | Embedded IdP | +| **OUK** | *Offline Update Kit* – signed tarball with images + feeds for air‑gap. | Admin guide #24 | +| **OTLP** | *OpenTelemetry Protocol* – exporter for traces & metrics. | `/metrics` endpoint | + +--- + +## P – S + +| Term | Definition | Notes | +|------|------------|-------| +| **P95** | 95th‑percentile latency metric. | Target ≤ 5 s SBOM path | +| **PDF SAR** | *Security Assessment Report* PDF produced by Pro edition. | Cosign‑signed | +| **Plug‑in** | Hot‑loadable DLL implementing a Stella contract (`IScannerRunner`, `ITlsProvider`, etc.). | Signed with Cosign | +| **Problem Details** | RFC 7807 JSON error format returned by API. | See API ref §0 | +| **Redis** | In‑memory datastore used for queue + cache. | Port 6379 | +| **Rekor** | Sigstore transparency log; future work for signature anchoring. | Road‑map P4 | +| **RPS** | *Requests Per Second*. | Backend perf budget 40 rps | +| **SBOM** | *Software Bill of Materials* – inventory of packages in an image. | Trivy JSON v2 | +| **Stella CLI** | Lightweight CLI that submits SBOMs for vulnerability scanning. | See CI recipes | +| **Seccomp** | Linux syscall filter JSON profile. | Backend shipped non‑root | +| **SLA** | *Service‑Level Agreement* – 24 h / 1‑ticket for Pro. | SRE runbook | +| **Span** | .NET ref‑like struct for zero‑alloc slicing. | Allowed with benchmarks | +| **Styker.NET** | Mutation testing runner used on critical libs. | Coverage ≥ 60 % | + +--- + +## T – Z + +| Term | Definition | Notes | +|------|------------|-------| +| **Trivy** | OSS CVE scanner powering the default `IScannerRunner`. | CLI pinned 0.64 | +| **Trivy‑srv** | Long‑running Trivy server exposing gRPC API; speeds up remote scans. | Variant A | +| **UI tile** | Dashboard element showing live metric (scans today, feed age, etc.). | Angular Signals | +| **WebSocket** | Full‑duplex channel (`/ws/scan`, `/ws/stats`) for UI real‑time. | Used by tiles | +| **Zastava** | Lightweight agent that inventories running containers and can enforce kills. | | + +--- + +### 11 Change log + +| Version | Date | Notes | +|---------|------|-------| +| **v1.0** | 2025‑07‑12 | First populated glossary – 52 terms covering Core docs. | + +*(End of Glossary v1.0)* diff --git a/docs/15_UI_GUIDE.md b/docs/15_UI_GUIDE.md new file mode 100755 index 00000000..5537246c --- /dev/null +++ b/docs/15_UI_GUIDE.md @@ -0,0 +1,234 @@ +#  15 - Pragmatic UI Guide --- **Stella Ops** + +# Stella Ops Web UI + +A fast, modular single‑page application for controlling scans, policies, offline updates and platform‑wide settings. +Built for sub‑second feedback, dark‑mode by default, and **no external CDNs** – everything ships inside the anonymous internal registry. + +--- + +## 0 Fast Facts + +| Aspect | Detail | +| ----------------- | -------------------------------------------------------------------------- | +| Tech Stack | **Angular {{ angular }}** + Vite dev server | +| Styling | **Tailwind CSS** | +| State | Angular Signals + RxJS | +| API Client | OpenAPI v3 generated services (Axios) | +| Auth | OAuth2 /OIDC (tokens from backend or external IdP) | +| i18n | JSON bundles – **`/locales/{lang}.json`** (English, Russian shipped) | +| Offline Updates 📌 | UI supports “OUK” tarball upload to refresh NVD / Trivy DB when air‑gapped | +| Build Artifacts | (`ui/dist/`) pushed to `registry.git.stella-ops.org/ui:${SHA}` | + +--- + +## 1 Navigation Map + +``` +Dashboard +└─ Scans + ├─ Active + ├─ History + └─ Reports +└─ Policies 📌 + ├─ Editor (YAML / Rego) 📌 + ├─ Import / Export 📌 + └─ History +└─ Settings + ├─ SBOM Format 📌 + ├─ Registry 📌 + ├─ Offline Updates (OUK) 📌 + ├─ Themes (Light / Dark / System) 📌 + └─ Advanced +└─ Plugins 🛠 +└─ Help / About +``` + +*The **Offline Updates (OUK)** node under **Settings** is new.* + +--- + +## 2 Technology Overview + +### 2.1 Build & Deployment + +1. `npm i && npm build` → generates `dist/` (~2.1 MB gzip). +2. A CI job tags and pushes the artifact as `ui:${GIT_SHA}` to the internal registry. +3. Backend serves static assets from `/srv/ui` (mounted from the image layer). + +_No external fonts or JS – true offline guarantee._ + +### 2.2 Runtime Boot + +1. **AppConfigService** pulls `/api/v1/config/ui` (contains feature flags, default theme, enabled plugins). +2. Locale JSON fetched (`/locales/{lang}.json`, falls back to `en`). +3. Root router mounts lazy‑loaded **feature modules** in the order supplied by backend – this is how future route plugins inject pages without forking the UI. + +--- + +## 3 Feature Walk‑Throughs + +### 3.1 Dashboard – Real‑Time Status + +* **Δ‑SBOM heat‑map** 📌 shows how many scans used delta mode vs. full unpack. +* “Feed Age” tile turns **orange** if NVD feed is older than 24 h; reverts after an **OUK** upload 📌. +* Live WebSocket updates for scans in progress (SignalR channel). +* **Quota Tile** – shows **Scans Today / {{ quota_token }}**; turns yellow at **≤ 10% remaining** (≈ 90% used), + red at {{ quota_token }} . +* **Token Expiry Tile** – shows days left on *client.jwt* (offline only); + turns orange at < 7 days. + +### 3.2 Scans Module + +| View | What you can do | +| ----------- | ------------------------------------------------------------------------------------------------- | +| **Active** | Watch progress bar (ETA ≤ 5 s) – newly added **Format** and **Δ** badges appear beside each item. | +| **History** | Filter by repo, tag, policy result (pass/block/soft‑fail). | +| **Reports** | Click row → HTML or PDF report rendered by backend (`/report/{digest}/html`). | + +### 3.3 📌 Policies Module (new) + +*Embedded **Monaco** editor with YAML + Rego syntax highlighting.* + +| Tab | Capability | +| ------------------- | ------------------------------------------------------------------------------------------------ | +| **Editor** | Write or paste `scan-policy.yaml` or inline Rego snippet. Schema validation shown inline. | +| **Import / Export** | Buttons map to `/policy/import` and `/policy/export`. Accepts `.yaml`, `.rego`, `.zip` (bundle). | +| **History** | Immutable audit log; diff viewer highlights rule changes. | + +#### 3.3.1 YAML → Rego Bridge + +If you paste YAML but enable **Strict Mode** (toggle), backend converts to Rego under the hood, stores both representations, and shows a side‑by‑side diff. + +### 3.4 📌 Settings Enhancements + +| Setting | Details | +| --------------------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| **SBOM Format** | Dropdown – *Trivy JSON*, *SPDX JSON*, *CycloneDX JSON*. | +| **Registry** | Displays pull URL (`registry.git.stella-ops.ru`) and Cosign key fingerprint. | +| **Offline Updates (OUK)** 📌 | Upload **`ouk*.tar.gz`** produced by the Offline Update Kit CLI. Backend unpacks, verifies SHA‑256 checksum & Cosign signature, then reloads Redis caches without restart. | +| **Theme** | Light, Dark, or Auto (system). | + +#### 3.4.1 OUK Upload Screen 📌 + +*Page path:* **Settings → Offline Updates (OUK)** +*Components:* + +1. **Drop Zone** – drag or select `.tar.gz` (max 1 GB). +2. **Progress Bar** – streaming upload with chunked HTTP. +3. **Verification Step** – backend returns status: + * *Signature valid* ✔️ + * *Digest mismatch* ❌ +4. **Feed Preview** – table shows *NVD date*, *OUI source build tag*, *CVE count delta*. +5. **Activate** – button issues `/feeds/activate/{id}`; on success the Dashboard “Feed Age” tile refreshes to green. +6. **History List** – previous OUK uploads with user, date, version; supports rollback. + +*All upload actions are recorded in the Policies → History audit log as type `ouk_update`.* + +### 3.5 Plugins Panel 🛠 (ships after UI modularisation) + +Lists discovered UI plugins; each can inject routes/panels. Toggle on/off without reload. + +### 3.6 Settings → **Quota & Tokens** (new) + +* View current **Client‑JWT claims** (tier, maxScansPerDay, expiry). +* **Generate Offline Token** – admin‑only button → POST `/token/offline` (UI wraps the API). +* Upload new token file for manual refresh. + +--- + +## 4 i18n & l10n + +* JSON files under `/locales`. +* Russian (`ru`) ships first‑class, translated security terms align with **GOST R ISO/IEC 27002‑2020**. +* “Offline Update Kit” surfaces as **“Оффлайн‑обновление базы уязвимостей”** in Russian locale. +* Community can add locales by uploading a new JSON via Plugins Panel once 🛠 ships. + +--- + +## 5 Accessibility + +* WCAG 2.1 AA conformance targeted. +* All color pairs pass contrast (checked by `vite-plugin-wcag`). +* Keyboard navigation fully supported; focus outlines visible in both themes. + +--- + +## 6 Theming 📌 + +| Layer | How to change | +| --------------- | ------------------------------------------------------------ | +| Tailwind | Palette variables under `tailwind.config.js > theme.colors`. | +| Runtime toggle | Stored in `localStorage.theme`, synced across tabs. | +| Plugin override | Future route plugins may expose additional palettes 🛠. | + +--- + +## 7 Extensibility Hooks + +| Area | Contract | Example | +| ------------- | ---------------------------------------- | ---------------------------------------------- | +| New route | `window.stella.registerRoute()` | “Secrets” scanner plugin adds `/secrets` page. | +| External link | `window.stella.addMenuLink(label, href)` | “Docs” link opens corporate Confluence. | +| Theme | `window.stella.registerTheme()` | High‑contrast palette for accessibility. | + +--- + +## 8 Road‑Map Tags + +| Feature | Status | +| ------------------------- | ------ | +| Policy Editor (YAML) | ✅ | +| Inline Rego validation | 🛠 | +| OUK Upload UI | ✅ | +| Plugin Marketplace UI | 🚧 | +| SLSA Verification banner | 🛠 | +| Rekor Transparency viewer | 🚧 | + +--- + +## 9 Non‑Commercial Usage Rules 📌 + +*(Extracted & harmonised from the Russian UI help page so that English docs remain licence‑complete.)* + +1. **Free for internal security assessments.** +2. Commercial resale or SaaS re‑hosting **prohibited without prior written consent** under AGPL §13. +3. If you distribute a fork **with UI modifications**, you **must**: + * Make the complete source code (including UI assets) publicly available. + * Retain original project attribution in footer. +4. All dependencies listed in `ui/package.json` remain under their respective OSS licences (MIT, Apache 2.0, ISC). +5. Use in government‑classified environments must comply with**applicable local regulations** governing cryptography and software distribution. + +--- + +## 10 Troubleshooting Tips + +| Symptom | Cause | Remedy | +| ----------------------------------- | ----------------------------------- | ----------------------------------------------------------------- | +| **White page** after login | `ui/dist/` hash mismatch | Clear browser cache; backend auto‑busts on version change. | +| Policy editor shows “Unknown field” | YAML schema drift | Sync your policy file to latest sample in *Settings → Templates*. | +| **OUK upload fails** at 99 % | Tarball built with outdated OUK CLI | Upgrade CLI (`ouk --version`) and rebuild package. | +| Icons look broken in Safari | *SVG `mask` unsupported* | Use Safari 17+ or switch to PNG icon set in Settings > Advanced. | + +--- + +## 11 Contributing + +* Run `npm dev` and open `http://localhost:5173`. +* Ensure `ng lint` and `ng test` pass before PR. +* Sign the **DCO** in your commit footer (`Signed-off-by`). + +--- + +## 12 Change Log + +| Version | Date | Highlights | +| ------- | ---------- | +| v2.4 | 2025‑07‑15 | **Added full OUK Offline Update upload flow** – navigation node, Settings panel, dashboard linkage, audit hooks. | +| v2.3 | 2025‑07‑14 | Added Policies module, SBOM Format & Registry settings, theming toggle, Δ‑SBOM indicators, extracted non‑commercial usage rules. | +| v2.2 | 2025‑07‑12 | Added user tips/workflows, CI notes, DevSecOps section, troubleshooting, screenshots placeholders. | +| v2.1 | 2025‑07‑12 | Removed PWA/Service‑worker; added oidc‑client‑ts; simplified roadmap | +| v2.0 | 2025‑07‑12 | Accessibility, Storybook, perf budgets, security rules | +| v1.1 | 2025‑07‑11 | Original OSS‑only guide | + +(End of Pragmatic UI Guide v2.2) diff --git a/docs/17_SECURITY_HARDENING_GUIDE.md b/docs/17_SECURITY_HARDENING_GUIDE.md new file mode 100755 index 00000000..92d33a10 --- /dev/null +++ b/docs/17_SECURITY_HARDENING_GUIDE.md @@ -0,0 +1,201 @@ +#  17 · Security Hardening Guide — **Stella Ops** +*(v2.0 — 12 Jul 2025)* + +> **Audience** — Site‑reliability and platform teams deploying **the open‑source Core** in production or restricted networks. +--- + +##  0 Table of Contents + +1. Threat model (summary) +2. Host‑OS baseline +3. Container & runtime hardening +4. Network‑plane guidance +5. Secrets & key management +6. Image, SBOM & plug‑in supply‑chain controls +7. Logging, monitoring & audit +8. Update & patch strategy +9. Incident‑response workflow +10. Pen‑testing & continuous assurance +11. Contacts & vulnerability disclosure +12. Change log + +--- + +##  1 Threat model (summary) + +| Asset | Threats | Mitigations | +| -------------------- | --------------------- | ---------------------------------------------------------------------- | +| SBOMs & scan results | Disclosure, tamper | TLS‑in‑transit, read‑only Redis volume, RBAC, Cosign‑verified plug‑ins | +| Backend container | RCE, code‑injection | Distroless image, non‑root UID, read‑only FS, seccomp + `CAP_DROP:ALL` | +| Update artefacts | Supply‑chain attack | Cosign‑signed images & SBOMs, enforced by admission controller | +| Admin credentials | Phishing, brute force | OAuth 2.0 with 12‑h token TTL, optional mTLS | + +--- + +##  2 Host‑OS baseline checklist + +| Item | Recommended setting | +| ------------- | --------------------------------------------------------- | +| OS | Ubuntu 22.04 LTS (kernel ≥ 5.15) or Alma 9 | +| Patches | `unattended‑upgrades` or vendor‑equivalent enabled | +| Filesystem | `noexec,nosuid` on `/tmp`, `/var/tmp` | +| Docker Engine | v24.*, API socket root‑owned (`0660`) | +| Auditd | Watch `/etc/docker`, `/usr/bin/docker*` and Compose files | +| Time sync | `chrony` or `systemd‑timesyncd` | + +--- + +##  3 Container & runtime hardening + +###  3.1 Docker Compose reference (`compose-core.yml`) + +```yaml +services: + backend: + image: registry.stella-ops.org/stella-ops/stella-ops: + user: "101:101" # non‑root + read_only: true + security_opt: + - "no-new-privileges:true" + - "seccomp:./seccomp-backend.json" + cap_drop: [ALL] + tmpfs: + - /tmp:size=64m,exec,nosymlink + environment: + - ASPNETCORE_URLS=https://+:8080 + - TLSPROVIDER=OpenSslGost + depends_on: [redis] + networks: [core-net] + healthcheck: + test: ["CMD", "wget", "-qO-", "https://localhost:8080/health"] + interval: 30s + timeout: 5s + retries: 5 + + redis: + image: redis:7.2-alpine + command: ["redis-server", "--requirepass", "${REDIS_PASS}", "--rename-command", "FLUSHALL", ""] + user: "redis" + read_only: true + cap_drop: [ALL] + tmpfs: + - /data + networks: [core-net] + +networks: + core-net: + driver: bridge +``` + +No dedicated “Redis” or “Mongo” sub‑nets are declared; the single bridge network suffices for the default stack. + +###  3.2 Kubernetes deployment highlights + +Use a separate NetworkPolicy that only allows egress from backend to Redis :6379. +securityContext: runAsNonRoot, readOnlyRootFilesystem, allowPrivilegeEscalation: false, drop all capabilities. +PodDisruptionBudget of minAvailable: 1. +Optionally add CosignVerified=true label enforced by an admission controller (e.g. Kyverno or Connaisseur). + +##  4 Network‑plane guidance + +| Plane | Recommendation | +| ------------------ | -------------------------------------------------------------------------- | +| North‑south | Terminate TLS 1.2+ (OpenSSL‑GOST default). Use LetsEncrypt or internal CA. | +| East‑west | Compose bridge or K8s ClusterIP only; no public Redis/Mongo ports. | +| Ingress controller | Limit methods to GET, POST, PATCH (no TRACE). | +| Rate‑limits | 40 rps default; tune ScannerPool.Workers and ingress limit‑req to match. | + +##  5 Secrets & key management + +| Secret | Storage | Rotation | +| --------------------------------- | ---------------------------------- | ----------------------------- | +| **Client‑JWT (offline)** | `/var/lib/stella/tokens/client.jwt` (root : 600) | **30 days** – provided by each OUK | +| REDIS_PASS | Docker/K8s secret | 90 days | +| OAuth signing key | /keys/jwt.pem (read‑only mount) | 180 days | +| Cosign public key | /keys/cosign.pub baked into image; | change on every major release | +| Trivy DB mirror token (if remote) | Secret + read‑only | 30 days | + +Never bake secrets into images; always inject at runtime. + +> **Operational tip:** schedule a cron reminding ops 5 days before +> `client.jwt` expiry. The backend also emits a Prometheus metric +> `stella_quota_token_days_remaining`. + +##  6 Image, SBOM & plug‑in supply‑chain controls + +* Images — Pull by digest not latest; verify: + +```bash +cosign verify ghcr.io/stellaops/backend@sha256: \ + --key https://stella-ops.org/keys/cosign.pub +``` + +* SBOM — Each release ships an SPDX file; store alongside images for audit. +* Third‑party plug‑ins — Place in /plugins/; backend will: +* Validate Cosign signature. +* Check [StellaPluginVersion("major.minor")]. +* Refuse to start if Security.DisablePluginUnsigned=false (default). + +##  7 Logging, monitoring & audit + +| Control | Implementation | +| ------------ | ----------------------------------------------------------------- | +| Log format | Serilog JSON; ship via Fluent‑Bit to ELK or Loki | +| Metrics | Prometheus /metrics endpoint; default Grafana dashboard in infra/ | +| Audit events | Redis stream audit; export daily to SIEM | +| Alert rules | Feed age  ≥ 48 h, P95 wall‑time > 5 s, Redis used memory > 75 % | + +###  7.1 Feedser authorization audits + +- Enable the Authority integration for Feedser (`authority.enabled=true`). Keep + `authority.allowAnonymousFallback` set to `true` only during migration and plan + to disable it before **2025-12-31 UTC** so the `/jobs*` surface always demands + a bearer token. +- Store the Authority client secret using Docker/Kubernetes secrets and point + `authority.clientSecretFile` at the mounted path; the value is read at startup + and never logged. +- Watch the `Feedser.Authorization.Audit` logger. Each entry contains the HTTP + status, subject, client ID, scopes, remote IP, and a boolean `bypass` flag + showing whether a network bypass CIDR allowed the request. Configure your SIEM + to alert when unauthenticated requests (`status=401`) appear with + `bypass=true`, or when unexpected scopes invoke job triggers. + +##  8 Update & patch strategy + +| Layer | Cadence | Method | +| -------------------- | -------------------------------------------------------- | ------------------------------ | +| Backend & CLI images | Monthly or CVE‑driven docker pull + docker compose up -d | +| Trivy DB | 24 h scheduler via Feedser (vulnerability ingest/merge/export service) | configurable via Feedser scheduler options | +| Docker Engine | vendor LTS | distro package manager | +| Host OS | security repos enabled | unattended‑upgrades | + +##  9 Incident‑response workflow + +* Detect — PagerDuty alert from Prometheus or SIEM. +* Contain — Stop affected Backend container; isolate Redis RDB snapshot. +* Eradicate — Pull verified images, redeploy, rotate secrets. +* Recover — Restore RDB, replay SBOMs if history lost. +* Review — Post‑mortem within 72 h; create follow‑up issues. +* Escalate P1 incidents to (24 × 7). + + +##  10 Pen‑testing & continuous assurance + +| Control | Frequency | Tool/Runner | +|----------------------|-----------------------|-------------------------------------------| +| OWASP ZAP baseline | Each merge to `main` | GitHub Action `zap-baseline-scan` | +| Dependency scanning | Per pull request | Trivy FS + Dependabot | +| External red‑team | Annual or pre‑GA | CREST‑accredited third‑party | + +##  11 Vulnerability disclosure & contact + +* Preferred channel: security@stella‑ops.org (GPG key on website). +* Coordinated disclosure reward: public credit and swag (no monetary bounty at this time). + +##  12 Change log + +| Version | Date | Notes | +| ------- | ---------- | -------------------------------------------------------------------------------------------------------------------------------- | +| v2.0 | 2025‑07‑12 | Full overhaul: host‑OS baseline, supply‑chain signing, removal of unnecessary sub‑nets, role‑based contact e‑mail, K8s guidance. | +| v1.1 | 2025‑07‑09 | Minor fence fixes. | +| v1.0 | 2025‑07‑09 | Original draft. | diff --git a/docs/18_CODING_STANDARDS.md b/docs/18_CODING_STANDARDS.md new file mode 100755 index 00000000..50c28af5 --- /dev/null +++ b/docs/18_CODING_STANDARDS.md @@ -0,0 +1,169 @@ +#  18 · Coding Standards & Contributor Guide — **Stella Ops** +*(v2.0 — 12 Jul 2025 · supersedes v1.0)* + +> **Audience** — Anyone sending a pull‑request to the open‑source Core. +> **Goal** — Keep the code‑base small‑filed, plug‑in‑friendly, DI‑consistent, and instantly readable. + +--- + +##  0 Why read this? + +* Cuts review time → quicker merges. +* Guarantees code is **hot‑load‑safe** for run‑time plug‑ins. +* Prevents style churn and merge conflicts. + +--- + +##  1 High‑level principles + +1. **SOLID first** – especially Interface & Dependency Inversion. +2. **100‑line rule** – any file > 100 physical lines must be split or refactored. +3. **Contract‑level ownership** – public abstractions live in lightweight *Contracts* libraries; impl classes live in runtime projects. +4. **Single Composition Root** – all DI wiring happens in **`StellaOps.Web/Program.cs`** and in each plug‑in’s `IoCConfigurator`; nothing else calls `IServiceCollection.BuildServiceProvider`. +5. **No Service Locator** – constructor injection only; static `ServiceProvider` is banned. +6. **Fail‑fast startup** – configuration validated before the web‑host listens. +7. **Hot‑load compatible** – no static singletons that survive plug‑in unload; avoid `Assembly.LoadFrom` outside the built‑in plug‑in loader. + +--- + +##  2 Repository layout (flat, July‑2025)** + +```text +src/ +├─ backend/ +│ ├─ StellaOps.Web/ # ASP.NET host + composition root +│ ├─ StellaOps.Common/ # Serilog, Result, helpers +│ ├─ StellaOps.Contracts/ # DTO + interface contracts (no impl) +│ ├─ StellaOps.Configuration/ # Options + validation +│ ├─ StellaOps.Localization/ +│ ├─ StellaOps.PluginLoader/ # Cosign verify, hot‑load +│ ├─ StellaOps.Scanners.Trivy/ # First‑party scanner +│ ├─ StellaOps.TlsProviders.OpenSsl/ +│ └─ … (additional runtime projects) +├─ plugins-sdk/ # Templated contracts & abstractions +└─ frontend/ # Angular workspace +tests/ # Mirrors src structure 1‑to‑1 +``` + +There are no folders named “Module” and no nested solutions. + +##  3 Naming & style conventions + +| Element | Rule | Example | +| ------------------------------------------------------------------------------- | --------------------------------------- | ------------------------------- | +| Namespaces | File‑scoped, StellaOps. | namespace StellaOps.Scanners; | +| Interfaces | I prefix, PascalCase | IScannerRunner | +| Classes / records | PascalCase | ScanRequest, TrivyRunner | +| Private fields | camelCase (no leading underscore) | redisCache, httpClient | +| Constants | SCREAMING_SNAKE_CASE | const int MAX_RETRIES = 3; | +| Async methods | End with Async | Task ScanAsync() | +| File length | ≤ 100 lines incl. using & braces | enforced by dotnet format check | +| Using directives | Outside namespace, sorted, no wildcards | — | + +Static analyzers (.editorconfig, StyleCop.Analyzers package) enforce the above. + +##  4 Dependency‑injection policy + +Composition root – exactly one per process: + +```csharp +builder.Services + .AddStellaCore() // extension methods from each runtime project + .AddPluginLoader("/Plugins", cfg); // hot‑load signed DLLs +``` + +Plug‑ins register additional services via the IoCConfigurator convention described in the Plug‑in SDK Guide, §5. +Never resolve services manually (provider.GetService()) outside the composition root; tests may use WebApplicationFactory or ServiceProvider.New() helpers. +Scoped lifetime is default; singletons only for stateless, thread‑safe helpers. + +##  5 Project organisation rules + +Contracts vs. Runtime – public DTO & interfaces live in .Contracts; implementation lives in sibling project. +Feature folders – inside each runtime project group classes by use‑case, e.g. + +```text +├─ Scan/ +│ ├─ ScanService.cs +│ └─ ScanController.cs +├─ Feed/ +└─ Tls/ +``` + +Tests – mirror the structure under tests/ one‑to‑one; no test code inside production projects. + +##  6 C# language features + +Nullable reference types enabled. +record for immutable DTOs. +Pattern matching encouraged; avoid long switch‑cascades. +Span & Memory OK when perf‑critical, but measure first. +Use await foreach over manual paginator loops. + +##  7 Error‑handling template + +```csharp +public async Task PostScan([FromBody] ScanRequest req) +{ + if (!ModelState.IsValid) return BadRequest(ModelState); + + try + { + ScanResult result = await scanService.ScanAsync(req); + if (result.Quota != null) + { + Response.Headers.TryAdd("X-Stella-Quota-Remaining", result.Quota.Remaining.ToString()); + Response.Headers.TryAdd("X-Stella-Reset", result.Quota.ResetUtc.ToString("o")); + } + return Ok(result); + } +} +``` + +RFC 7807 ProblemDetails for all non‑200s. +Capture structured logs with Serilog’s message‑template syntax. + +##  8 Async & threading + +* All I/O is async; no .Result / .Wait(). +* Library code: ConfigureAwait(false). +* Limit concurrency via Channel or Parallel.ForEachAsync, never raw Task.Run loops. + +##  9 Testing rules + +| Layer | Framework | Coverage gate | +| ------------------------ | ------------------------ | -------------------------- | +| Unit | xUnit + FluentAssertions | ≥ 80 % line, ≥ 60 % branch | +| Integration | Testcontainers | Real Redis & Trivy | +| Mutation (critical libs) | Stryker.NET | ≥ 60 % score | + +One test project per runtime/contract project; naming .Tests. + +##  10 Static analysis & formatting + +* dotnet format must exit clean (CI gate). +* StyleCop.Analyzers + Roslyn‑Security‑Guard run on every PR. +* CodeQL workflow runs nightly on main. + +##  11 Commit & PR checklist + +* Conventional Commit prefix (feat:, fix:, etc.). +* dotnet format & dotnet test both green. +* Added or updated XML‑doc comments for public APIs. +* File count & length comply with 100‑line rule. +* If new public contract → update relevant markdown doc & JSON‑Schema. + +##  12 Common pitfalls + +|Symptom| Root cause | Fix +|-------|-------------|------------------- +|InvalidOperationException: Cannot consume scoped service...| Mis‑matched DI lifetimes| Use scoped everywhere unless truly stateless +|Hot‑reload plug‑in crash| Static singleton caching plugin types| Store nothing static; rely on DI scopes + +> 100‑line style violation |Large handlers or utils |Split into private helpers or new class + +##  13 Change log + +| Version | Date | Notes | +| ------- | ---------- | -------------------------------------------------------------------------------------------------- | +| v2.0 | 2025‑07‑12 | Updated DI policy, 100‑line rule, new repo layout, camelCase fields, removed “Module” terminology. | +| 1.0 | 2025‑07‑09 | Original standards. | diff --git a/docs/19_TEST_SUITE_OVERVIEW.md b/docs/19_TEST_SUITE_OVERVIEW.md new file mode 100755 index 00000000..f204be35 --- /dev/null +++ b/docs/19_TEST_SUITE_OVERVIEW.md @@ -0,0 +1,109 @@ +# Automated Test‑Suite Overview + +This document enumerates **every automated check** executed by the Stella Ops +CI pipeline, from unit level to chaos experiments. It is intended for +contributors who need to extend coverage or diagnose failures. + +> **Build parameters** – values such as `{{ dotnet }}` (runtime) and +> `{{ angular }}` (UI framework) are injected at build time. + +--- + +## Layer map + +| Layer | Tooling | Entry‑point | Frequency | +|-------|---------|-------------|-----------| +| **1. Unit** | `xUnit` (dotnet test) | `*.Tests.csproj` | per PR / push | +| **2. Property‑based** | `FsCheck` | `SbomPropertyTests` | per PR | +| **3. Integration (API)** | `Testcontainers` suite | `test/Api.Integration` | per PR + nightly | +| **4. Integration (DB-merge)** | in-memory Mongo + Redis | `Feedser.Integration` (vulnerability ingest/merge/export service) | per PR | +| **5. Contract (gRPC)** | `Buf breaking` | `buf.yaml` files | per PR | +| **6. Front‑end unit** | `Jest` | `ui/src/**/*.spec.ts` | per PR | +| **7. Front‑end E2E** | `Playwright` | `ui/e2e/**` | nightly | +| **8. Lighthouse perf / a11y** | `lighthouse-ci` (Chrome headless) | `ui/dist/index.html` | nightly | +| **9. Load** | `k6` scripted scenarios | `k6/*.js` | nightly | +| **10. Chaos CPU / OOM** | `pumba` | Docker Compose overlay | weekly | +| **11. Dependency scanning** | `Trivy fs` + `dotnet list package --vuln` | root | per PR | +| **12. License compliance** | `LicenceFinder` | root | per PR | +| **13. SBOM reproducibility** | `in‑toto attestation` diff | GitLab job | release tags | + +--- + +## Quality gates + +| Metric | Budget | Gate | +|--------|--------|------| +| API unit coverage | ≥ 85 % lines | PR merge | +| API response P95 | ≤ 120 ms | nightly alert | +| Δ‑SBOM warm scan P95 (4 vCPU) | ≤ 5 s | nightly alert | +| Lighthouse performance score | ≥ 90 | nightly alert | +| Lighthouse accessibility score | ≥ 95 | nightly alert | +| k6 sustained RPS drop | < 5 % vs baseline | nightly alert | + +--- + +## Local runner + +```bash +# minimal run: unit + property + frontend tests +./scripts/dev-test.sh + +# full stack incl. Playwright and lighthouse +./scripts/dev-test.sh --full +```` + +The script spins up MongoDB/Redis via Testcontainers and requires: + +* Docker ≥ 25 +* Node 20 (for Jest/Playwright) + +--- + +### Feedser OSV↔GHSA parity fixtures + +The Feedser connector suite includes a regression test (`OsvGhsaParityRegressionTests`) +that checks a curated set of GHSA identifiers against OSV responses. The fixture +snapshots live in `src/StellaOps.Feedser.Source.Osv.Tests/Fixtures/` and are kept +deterministic so the parity report remains reproducible. + +To refresh the fixtures when GHSA/OSV payloads change: + +1. Ensure outbound HTTPS access to `https://api.osv.dev` and `https://api.github.com`. +2. Run `UPDATE_PARITY_FIXTURES=1 dotnet test src/StellaOps.Feedser.Source.Osv.Tests/StellaOps.Feedser.Source.Osv.Tests.csproj`. +3. Commit the regenerated `osv-ghsa.*.json` files that the test emits (raw snapshots and canonical advisories). + +The regen flow logs `[Parity]` messages and normalises `recordedAt` timestamps so the +fixtures stay stable across machines. + +--- + +## CI job layout + +```mermaid +flowchart LR + subgraph fast-path + U[xUnit] --> P[FsCheck] --> I1[Testcontainer API] + end + + I1 --> FE[Jest] + FE --> E2E[Playwright] + E2E --> Lighthouse + Lighthouse --> INTEG2[Feedser] + INTEG2 --> LOAD[k6] + LOAD --> CHAOS[pumba] + CHAOS --> RELEASE[Attestation diff] +``` + +--- + +## Adding a new test layer + +1. Extend `scripts/dev-test.sh` so local contributors get the layer by default. +2. Add a dedicated GitLab job in `.gitlab-ci.yml` (stage `test` or `nightly`). +3. Register the job in `docs/19_TEST_SUITE_OVERVIEW.md` *and* list its metric + in `docs/metrics/README.md`. + +--- + +*Last updated {{ "now" | date: "%Y‑%m‑%d" }}* + diff --git a/docs/21_INSTALL_GUIDE.md b/docs/21_INSTALL_GUIDE.md new file mode 100755 index 00000000..310e44f4 --- /dev/null +++ b/docs/21_INSTALL_GUIDE.md @@ -0,0 +1,169 @@ +# Stella Ops — Installation Guide (Docker & Air‑Gap) + + + +> **Status — public α not yet published.** +> The commands below will work as soon as the first image is tagged +> `registry.stella-ops.org/stella-ops/stella-ops:0.1.0-alpha` +> (target date: **late 2025**). Track progress on the +> [road‑map](/roadmap/). + +--- + +## 0 · Prerequisites + +| Item | Minimum | Notes | +|------|---------|-------| +| Linux | Ubuntu 22.04 LTS / Alma 9 | x86‑64 or arm64 | +| CPU / RAM | 2 vCPU / 2 GiB | Laptop baseline | +| Disk | 10 GiB SSD | SBOM + vuln DB cache | +| Docker | **Engine 25 + Compose v2** | `docker -v` | +| TLS | OpenSSL 1.1 +  | Self‑signed cert generated at first run | + +--- + +## 1 · Connected‑host install (Docker Compose) + +```bash +# 1. Make a working directory +mkdir stella && cd stella + +# 2. Download the signed Compose bundle + example .env +curl -LO https://get.stella-ops.org/releases/latest/.env.example +curl -LO https://get.stella-ops.org/releases/latest/.env.example.sig +curl -LO https://get.stella-ops.org/releases/latest/docker-compose.infrastructure.yml +curl -LO https://get.stella-ops.org/releases/latest/docker-compose.infrastructure.yml.sig +curl -LO https://get.stella-ops.org/releases/latest/docker-compose.stella-ops.yml +curl -LO https://get.stella-ops.org/releases/latest/docker-compose.stella-ops.yml.sig + +# 3. Verify provenance (Cosign public key is stable) +cosign verify-blob \ + --key https://stella-ops.org/keys/cosign.pub \ + --signature .env.example.sig \ + .env.example + +cosign verify-blob \ + --key https://stella-ops.org/keys/cosign.pub \ + --signature docker-compose.infrastructure.yml.sig \ + docker-compose.infrastructure.yml + +cosign verify-blob \ + --key https://stella-ops.org/keys/cosign.pub \ + --signature docker-compose.stella-ops.yml.sig \ + docker-compose.stella-ops.yml + +# 4. Copy .env.example → .env and edit secrets +cp .env.example .env +$EDITOR .env + +# 5. Launch databases (MongoDB + Redis) +docker compose --env-file .env -f docker-compose.infrastructure.yml up -d + +# 6. Launch Stella Ops (first run pulls ~50 MB merged vuln DB) +docker compose --env-file .env -f docker-compose.stella-ops.yml up -d +```` + +*Default login:* `admin / changeme` +UI: [https://\<host\>:8443](https://<host>:8443) (self‑signed certificate) + +> **Pinning best‑practice** – in production environments replace +> `stella-ops:latest` with the immutable digest printed by +> `docker images --digests`. + +### 1.1 · Feedser authority configuration + +The Feedser container reads configuration from `etc/feedser.yaml` plus +`FEEDSER_` environment variables. To enable the new Authority integration: + +1. Add the following keys to `.env` (replace values for your environment): + + ```bash + FEEDSER_AUTHORITY__ENABLED=true + FEEDSER_AUTHORITY__ALLOWANONYMOUSFALLBACK=true # temporary rollout only + FEEDSER_AUTHORITY__ISSUER="https://authority.internal" + FEEDSER_AUTHORITY__AUDIENCES__0="api://feedser" + FEEDSER_AUTHORITY__REQUIREDSCOPES__0="feedser.jobs.trigger" + FEEDSER_AUTHORITY__CLIENTID="feedser-jobs" + FEEDSER_AUTHORITY__CLIENTSECRETFILE="/run/secrets/feedser_authority_client" + FEEDSER_AUTHORITY__BYPASSNETWORKS__0="127.0.0.1/32" + FEEDSER_AUTHORITY__BYPASSNETWORKS__1="::1/128" + ``` + + Store the client secret outside source control (Docker secrets, mounted file, + or Kubernetes Secret). Feedser loads the secret during post-configuration, so + the value never needs to appear in the YAML template. + +2. Redeploy Feedser: + + ```bash + docker compose --env-file .env -f docker-compose.stella-ops.yml up -d feedser + ``` + +3. Tail the logs: `docker compose logs -f feedser`. Successful `/jobs*` calls now + emit `Feedser.Authorization.Audit` entries listing subject, client ID, scopes, + remote IP, and whether the bypass CIDR allowed the call. 401 denials always log + `bypassAllowed=false` so unauthenticated cron jobs are easy to catch. + +> **Enforcement deadline** – keep `FEEDSER_AUTHORITY__ALLOWANONYMOUSFALLBACK=true` +> only while validating the rollout. Set it to `false` (and restart Feedser) +> before **2025-12-31 UTC** to require tokens in production. + +--- + +## 2 · Optional: request a free quota token + +Anonymous installs allow **{{ quota\_anon }} scans per UTC day**. +Email `token@stella-ops.org` to receive a signed JWT that raises the limit to +**{{ quota\_token }} scans/day**. Insert it into `.env`: + +```bash +STELLA_JWT="paste‑token‑here" +docker compose --env-file .env -f docker-compose.stella-ops.yml \ + exec stella-ops stella set-jwt "$STELLA_JWT" +``` + +>  The UI shows a reminder at 200 scans and throttles above the limit but will +>  **never block** your pipeline. + +--- + +## 3 · Air‑gapped install (Offline Update Kit) + +When running on an isolated network use the **Offline Update Kit (OUK)**: + +```bash +# Download & verify on a connected host +curl -LO https://get.stella-ops.org/ouk/stella-ops-offline-kit-v0.1a.tgz +curl -LO https://get.stella-ops.org/ouk/stella-ops-offline-kit-v0.1a.tgz.sig + +cosign verify-blob \ + --key https://stella-ops.org/keys/cosign.pub \ + --signature stella-ops-offline-kit-v0.1a.tgz.sig \ + stella-ops-offline-kit-v0.1a.tgz + +# Transfer → air‑gap → import +docker compose --env-file .env -f docker-compose.stella-ops.yml \ + exec stella admin import-offline-usage-kit stella-ops-offline-kit-v0.1a.tgz +``` + +*Import is atomic; no service downtime.* + +For details see the dedicated [Offline Kit guide](/offline/). + +--- + +## 4 · Next steps + +* **5‑min Quick‑Start:** `/quickstart/` +* **CI recipes:** `docs/ci/20_CI_RECIPES.md` +* **Plug‑in SDK:** `/plugins/` + +--- + +*Generated {{ "now" | date: "%Y‑%m‑%d" }} — build tags inserted at render time.* diff --git a/docs/23_FAQ_MATRIX.md b/docs/23_FAQ_MATRIX.md new file mode 100755 index 00000000..c4636538 --- /dev/null +++ b/docs/23_FAQ_MATRIX.md @@ -0,0 +1,61 @@ +# Stella Ops — Frequently Asked Questions (Matrix) + +## Quick glance + +| Question | Short answer | +|----------|----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| What is Stella Ops? | A lightning‑fast, SBOM‑first container‑security scanner written in **.NET {{ dotnet }}** with an **Angular {{ angular }}** web UI. | +| How fast is it? | Warm scans finish in **\< 5 s** on a 4‑vCPU runner; first scans stay **\< 30 s**. | +| Is it free? | Yes – **{{ quota_anon }} scans / day** anonymously. Requesting a free JWT lifts the limit to **{{ quota_token }}**. A gentle reminder shows at 200; exceeding the cap throttles speed but never blocks. | +| Does it run offline? | Yes — download the signed **Offline Update Kit**; see `/offline/`. | +| Can I extend it? | Yes — restart‑time plug‑ins (`ISbomMutator`, `IVulnerabilityProvider`, `IResultSink`, OPA Rego). Marketplace GA in v1.0. | + +--- + +## Road‑map (authoritative link) + +The full, always‑up‑to‑date roadmap lives at . +Snapshot: + +| Version | Target date | Locked‑in scope (freeze at β) | +|---------|-------------|--------------------------------| +| **v0.1 α** | *Late 2025* | Δ‑SBOM engine, nightly re‑scan, Offline Kit v1, {{ quota_anon }}/ {{ quota_token }} quota | +| **v0.2 β** | Q1 2026 | *Zastava* forbidden‑image scanner, registry sweeper, SDK β | +| **v0.3 β** | Q2 2026 | YAML/Rego policy‑as‑code, SARIF output, OUK auto‑import | +| **v0.4 RC** | Q3 2026 | AI remediation advisor, LDAP/AD SSO, pluggable TLS providers | +| **v1.0 GA** | Q4 2026 | SLSA L3 provenance, signed plug‑in marketplace | + +--- + +## Technical matrix + +| Category | Detail | +|----------|--------| +| **Core runtime** | C# 14 on **.NET {{ dotnet }}** | +| **UI stack** | **Angular {{ angular }}** + TailwindCSS | +| **Container base** | Distroless glibc (x86‑64 & arm64) | +| **Data stores** | MongoDB 7 (SBOM + findings), Redis 7 (LRU cache + quota) | +| **Release integrity** | Cosign‑signed images & TGZ, reproducible build, SPDX 2.3 SBOM | +| **Extensibility** | Plug‑ins in any .NET language (restart load); OPA Rego policies | +| **Default quotas** | Anonymous **{{ quota_anon }} scans/day** · JWT **{{ quota_token }}** | + +--- + +## Quota enforcement (overview) + +* Counters live in Redis with 24 h keys: `quota:ip:` or `quota:tid:`. +* Soft reminder banner at 200 daily scans. +* Past the limit: first 30 excess requests delayed 5 s; afterwards 60 s. +* Behaviour is identical online and offline (validation local). + +For full flow see `docs/30_QUOTA_ENFORCEMENT_FLOW1.md`. + +--- + +## Further reading + +* **Install guide:** `/install/` +* **Offline mode:** `/offline/` +* **Security policy:** `/security/` +* **Governance:** `/governance/` +* **Community chat:** Matrix `#stellaops:libera.chat` diff --git a/docs/24_OFFLINE_KIT.md b/docs/24_OFFLINE_KIT.md new file mode 100755 index 00000000..156beb90 --- /dev/null +++ b/docs/24_OFFLINE_KIT.md @@ -0,0 +1,94 @@ +# Offline Update Kit (OUK) — Air‑Gap Bundle + + + +The **Offline Update Kit** packages everything Stella Ops needs to run on a +completely isolated network: + +| Component | Contents | +|-----------|----------| +| **Merged vulnerability feeds** | OSV, GHSA plus optional NVD 2.0, CNNVD, CNVD, ENISA, JVN and BDU | +| **Container images** | `stella-ops`, *Zastava* sidecar (x86‑64 & arm64) | +| **Provenance** | Cosign signature, SPDX 2.3 SBOM, in‑toto SLSA attestation | +| **Delta patches** | Daily diff bundles keep size \< 350 MB | + +*Scanner core:* C# 12 on **.NET {{ dotnet }}**. +*Imports are idempotent and atomic — no service downtime.* + +--- + +## 1 · Download & verify + +```bash +curl -LO https://get.stella-ops.org/ouk/stella-ops-offline-kit-.tgz +curl -LO https://get.stella-ops.org/ouk/stella-ops-offline-kit-.tgz.sig + +cosign verify-blob \ + --key https://stella-ops.org/keys/cosign.pub \ + --signature stella-ops-offline-kit-.tgz.sig \ + stella-ops-offline-kit-.tgz +```` + +Verification prints **OK** and the SHA‑256 digest; cross‑check against the +[changelog](https://git.stella-ops.org/stella-ops/offline-kit/-/releases). + +--- + +## 2 · Import on the air‑gapped host + +```bash +docker compose --env-file .env \ + -f docker-compose.stella-ops.yml \ + exec stella-ops \ + stella admin import-offline-usage-kit stella-ops-offline-kit-.tgz +``` + +* The CLI validates the Cosign signature **before** activation. +* Old feeds are kept until the new bundle is fully verified. +* Import time on a SATA SSD: ≈ 25 s for a 300 MB kit. + +--- + +## 3 · Delta patch workflow + +1. **Connected site** fetches `stella-ouk-YYYY‑MM‑DD.delta.tgz`. +2. Transfer via any medium (USB, portable disk). +3. `stella admin import-offline-usage-kit ` applies only changed CVE rows & images. + +Daily deltas are **< 30 MB**; weekly roll‑up produces a fresh full kit. + +--- + +## 4 · Quota behaviour offline + +The scanner enforces the same fair‑use limits offline: + +* **Anonymous:** {{ quota\_anon }} scans per UTC day +* **Free JWT:** {{ quota\_token }} scans per UTC day + +Soft reminder at 200 scans; throttle above the ceiling but **never block**. +See the detailed rules in +[`33_333_QUOTA_OVERVIEW.md`](33_333_QUOTA_OVERVIEW.md). + +--- + +## 5 · Troubleshooting + +| Symptom | Explanation | Fix | +| -------------------------------------- | ---------------------------------------- | ------------------------------------- | +| `could not verify SBOM hash` | Bundle corrupted in transit | Re‑download / re‑copy | +| Import hangs at `Applying feeds…` | Low disk space in `/var/lib/stella` | Free ≥ 2 GiB before retry | +| `quota exceeded` same day after import | Import resets counters at UTC 00:00 only | Wait until next UTC day or load a JWT | + +--- + +## 6 · Related documentation + +* **Install guide:** `/install/#air-gapped` +* **Sovereign mode rationale:** `/sovereign/` +* **Security policy:** `/security/#reporting-a-vulnerability` diff --git a/docs/29_LEGAL_FAQ_QUOTA.md b/docs/29_LEGAL_FAQ_QUOTA.md new file mode 100755 index 00000000..28a69c5b --- /dev/null +++ b/docs/29_LEGAL_FAQ_QUOTA.md @@ -0,0 +1,84 @@ +# Legal FAQ — Free‑Tier Quota & AGPL Compliance + +> **Operational behaviour (limits, counters, delays) is documented in +> [`33_333_QUOTA_OVERVIEW.md`](33_333_QUOTA_OVERVIEW.md).** +> This page covers only the legal aspects of offering Stella Ops as a +> service or embedding it into another product while the free‑tier limits are +> in place. + +--- + +## 1 · Does enforcing a quota violate the AGPL? + +**No.** +AGPL‑3.0 does not forbid implementing usage controls in the program itself. +Recipients retain the freedoms to run, study, modify and share the software. +The Stella Ops quota: + +* Is enforced **solely at the service layer** (Redis counters) — the source + code implementing the quota is published under AGPL‑3.0‑or‑later. +* Never disables functionality; it introduces *time delays* only after the + free allocation is exhausted. +* Can be bypassed entirely by rebuilding from source and removing the + enforcement middleware — the licence explicitly allows such modifications. + +Therefore the quota complies with §§ 0 & 2 of the AGPL. + +--- + +## 2 · Can I redistribute Stella Ops with the quota removed? + +Yes, provided you: + +1. **Publish the full corresponding source code** of your modified version + (AGPL § 13 & § 5c), and +2. Clearly indicate the changes (AGPL § 5a). + +You may *retain* or *relax* the limits, or introduce your own tiering, as long +as the complete modified source is offered to every user of the service. + +--- + +## 3 · Embedding in a proprietary appliance + +You may ship Stella Ops inside a hardware or virtual appliance **only if** the +entire combined work is distributed under **AGPL‑3.0‑or‑later** and you supply +the full source code for both the scanner and your integration glue. + +Shipping an AGPL component while keeping the rest closed‑source violates +§ 13 (*“remote network interaction”*). + +--- + +## 4 · SaaS redistribution + +Operating a public SaaS that offers Stella Ops scans to third parties triggers +the **network‑use clause**. You must: + +* Provide the complete, buildable source of **your running version** — + including quota patches or UI branding. +* Present the offer **conspicuously** (e.g. a “Source Code” footer link). + +Failure to do so breaches § 13 and can terminate your licence under § 8. + +--- + +## 5 · Is e‑mail collection for the JWT legal? + +* **Purpose limitation (GDPR Art. 5‑1 b):** address is used only to deliver the + JWT or optional release notes. +* **Data minimisation (Art. 5‑1 c):** no name, IP or marketing preferences are + required; a blank e‑mail body suffices. +* **Storage limitation (Art. 5‑1 e):** addresses are deleted or hashed after + ≤ 7 days unless the sender opts into updates. + +Hence the token workflow adheres to GDPR principles. + +--- + +## 6 · Change‑log + +| Version | Date | Notes | +|---------|------|-------| +| **2.0** | 2025‑07‑16 | Removed runtime quota details; linked to new authoritative overview. | +| 1.0 | 2024‑12‑20 | Initial legal FAQ. | diff --git a/docs/30_QUOTA_ENFORCEMENT_FLOW1.md b/docs/30_QUOTA_ENFORCEMENT_FLOW1.md new file mode 100755 index 00000000..480753f7 --- /dev/null +++ b/docs/30_QUOTA_ENFORCEMENT_FLOW1.md @@ -0,0 +1,93 @@ +# Quota Enforcement — Flow Diagram (rev 2.1) + +> **Scope** – this document explains *how* the free‑tier limits are enforced +> inside the scanner service. For policy rationale and legal aspects see +> [`33_333_QUOTA_OVERVIEW.md`](33_333_QUOTA_OVERVIEW.md). + +--- + +## 0 · Key parameters (rev 2.1) + +| Symbol | Value | Meaning | +|--------|-------|---------| +| `L_anon` | **{{ quota_anon }}** | Daily ceiling for anonymous users | +| `L_jwt` | **{{ quota_token }}** | Daily ceiling for token holders | +| `T_warn` | `200` | Soft reminder threshold | +| `D_soft` | `5 000 ms` | Delay for *first 30* over‑quota scans | +| `D_hard` | `60 000 ms` | Delay for all scans beyond the soft window | + +`L_active` is `L_jwt` if a valid token is present; else `L_anon`. + +--- + +## 1 · Sequence diagram + +```mermaid +sequenceDiagram + participant C as Client + participant API as Scanner API + participant REDIS as Redis (quota) + C->>API: /scan + API->>REDIS: INCR quota: + REDIS-->>API: new_count + alt new_count ≤ L_active + API-->>C: 202 Accepted (no delay) + else new_count ≤ L_active + 30 + API->>C: wait D_soft + API-->>C: 202 Accepted + else + API->>C: wait D_hard + API-->>C: 202 Accepted + end +```` + +*Counters auto‑expire **24 h** after first increment (00:00 UTC reset).* + +--- + +## 2 · Redis key layout + +| Key pattern | TTL | Description | +| ---------------------- | ---- | --------------------------------- | +| `quota:ip:` | 24 h | Anonymous quota per *hashed* IP | +| `quota:tid:` | 24 h | Token quota per *hashed* token‑ID | +| `quota:ip::ts` | 24 h | First‑seen timestamp (ISO 8601) | + +Keys share a common TTL for efficient mass expiry via `redis-cli --scan`. + +--- + +## 3 · Pseudocode (Go‑style) + +```go +func gate(key string, limit int) (delay time.Duration) { + cnt, _ := rdb.Incr(ctx, key).Result() + + switch { + case cnt <= limit: + return 0 // under quota + case cnt <= limit+30: + return 5 * time.Second + default: + return 60 * time.Second + } +} +``` + +*The middleware applies `time.Sleep(delay)` **before** processing the scan +request; it never returns `HTTP 429` under the free tier.* + +--- + +## 4 · Metrics & monitoring + +| Metric | PromQL sample | Alert | +| ------------------------------ | ------------------------------------------ | --------------------- | +| `stella_quota_soft_hits_total` | `increase(...[5m]) > 50` | Many users near limit | +| `stella_quota_hard_hits_total` | `rate(...[1h]) > 0.1` | Potential abuse | +| Average delay per request | `histogram_quantile(0.95, sum(rate(...)))` | P95 < 1 s expected | + +--- + + +*Generated {{ "now" | date: "%Y‑%m‑%d" }} — values pulled from central constants.* diff --git a/docs/33_333_QUOTA_OVERVIEW.md b/docs/33_333_QUOTA_OVERVIEW.md new file mode 100755 index 00000000..65691938 --- /dev/null +++ b/docs/33_333_QUOTA_OVERVIEW.md @@ -0,0 +1,120 @@ +# Free‑Tier Quota — **{{ quota_anon }}/ {{ quota_token }} Scans per UTC Day** + +Stella Ops is free for individual developers and small teams. +To avoid registry abuse the scanner enforces a **two‑tier daily quota** +— fully offline capable. + +| Mode | Daily ceiling | How to obtain | +|------|---------------|---------------| +| **Anonymous** | **{{ quota_anon }} scans** | No registration. Works online or air‑gapped. | +| **Free JWT token** | **{{ quota_token }} scans** | Email `token@stella-ops.org` (blank body). Bot replies with a signed JWT. | + +*Soft reminder banner appears at 200 scans. Exceeding the limit never blocks – +the CLI/UI introduce a delay, detailed below.* + +--- + +## 1 · Token structure + +```jsonc +{ + "iss": "stella-ops.org", + "sub": "free-tier", + "tid": "7d2285…", // 32‑byte random token‑ID + "tier": {{ quota_token }}, // daily scans allowed + "exp": 1767139199 // POSIX seconds (mandatory) – token expiry +} +```` + +* The **token‑ID (`tid`)** – not the e‑mail – is hashed *(SHA‑256 + salt)* + and stored for counter lookup. +* Verification uses the bundled public key (`keys/cosign.pub`) so **offline + hosts validate tokens locally**. An optional `exp` claim may be present; + if absent, the default is a far‑future timestamp used solely for schema + compatibility. + +--- + +## 2 · Enforcement algorithm (rev 2.1) + +| Step | Operation | Typical latency | +| ---- | ------------------------------------------------------------------------------ | ------------------------------------ | +| 1 | `key = sha256(ip)` *or* `sha256(tid)` | < 0.1 ms | +| 2 | `count = INCR quota:` in Redis (24 h TTL) | 0.2 ms (Lua) | +| 3 | If `count > limit` → `WAIT delay_ms` | first 30 × 5 000 ms → then 60 000 ms | +| 4 | Return HTTP 429 **only if** `delay > 60 s` (should never fire under free tier) | — | + +*Counters reset at **00:00 UTC**.* + +--- + +## 3 · CLI / API integration + +```bash +# Example .env +docker run --rm \ + -e DOCKER_HOST="$DOCKER_HOST" \ # remote‑daemon pointer + -v "$WORKSPACE/${SBOM_FILE}:/${SBOM_FILE}:ro" \ # mount SBOM under same name at container root + -e STELLA_OPS_URL="https://${STELLA_URL}" \ # where the CLI posts findings + "$STELLA_URL/registry/stella-cli:latest" \ + scan --sbom "/${SBOM_FILE}" "$IMAGE" +``` + +*No JWT? → scanner defaults to anonymous quota.* + +--- + +## 4 · Data retention & privacy + +| Data | Retention | Purpose | +| ---------------------- | ------------------------------------ | ---------------- | +| IP hash (`quota:ip:*`) | 7 days, then salted hash only | Abuse rate‑limit | +| Token‑ID hash | Until revoked | Counter lookup | +| E‑mail (token request) | ≤ 7 days unless newsletters opted‑in | Deliver the JWT | + +*No personal data leaves your infrastructure when running offline.* + +--- + +## 5 · Common questions + +
+What happens at exactly 200 scans? + +> The UI/CLI shows a yellow “fair‑use reminder”. +> No throttling is applied yet. +> Once you cross the full limit, the **first 30** over‑quota scans incur a +> 5‑second delay; further excess scans delay **60 s** each. + +
+ +
+Does the quota differ offline? + +> No. Counters are evaluated locally in Redis; the same limits apply even +> without Internet access. + +
+ +
+Can I reset counters manually? + +> Yes – delete the `quota:*` keys in Redis, but we recommend letting them +> expire at midnight to keep statistics meaningful. + +
+ +--- + +## 6 · Revision history + +| Version | Date | Notes | +| ------- | ---------- | ------------------------------------------------------------------- | +| **2.1** | 2025‑07‑16 | Consolidated into single source; delays re‑tuned (30 × 5 s → 60 s). | +|  2.0 | 2025‑04‑07 | Switched counters from Mongo to Redis. | +|  1.0 | 2024‑12‑20 | Initial free‑tier design. | + +--- + +**Authoritative source** — any doc or website section that references quotas +*must* link to this file instead of duplicating text. diff --git a/docs/40_ARCHITECTURE_OVERVIEW.md b/docs/40_ARCHITECTURE_OVERVIEW.md new file mode 100755 index 00000000..15b2c05c --- /dev/null +++ b/docs/40_ARCHITECTURE_OVERVIEW.md @@ -0,0 +1,133 @@ +# Stella Ops — High‑Level Architecture + + + +This document offers a birds‑eye view of how the major components interact, +why the system leans *monolith‑plus‑plug‑ins*, and where extension points live. + +> For a *timeline* of when features arrive, see the public +> [road‑map](/roadmap/) — no version details are repeated here. + +--- + +## 0 · Guiding principles + +| Principle | Rationale | +|-----------|-----------| +| **SBOM‑first** | Scan existing CycloneDX/SPDX if present; fall back to layer unpack. | +| **Δ‑processing** | Re‑analyse only changed layers; reduces P95 warm path to \< 5 s. | +| **All‑managed code** | Entire stack is 100 % managed (.NET / TypeScript); no `unsafe` blocks or native extensions — eases review and reproducible builds. | +| **Restart‑time plug‑ins** | Avoids the attack surface of runtime DLL injection; still allows custom scanners & exporters. | +| **Sovereign‑by‑design** | No mandatory outbound traffic; Offline Kit distributes feeds. | + +--- + +## 1 · Module graph + +```mermaid +graph TD + A(API Gateway) + B1(Scanner Core
.NET latest LTS) + B2(Feedser service\n(vuln ingest/merge/export)) + B3(Policy Engine OPA) + C1(Redis 7) + C2(MongoDB 7) + D(UI SPA
Angular latest version) + A -->|gRPC| B1 + B1 -->|async| B2 + B1 -->|OPA| B3 + B1 --> C1 + B1 --> C2 + A -->|REST/WS| D +```` + +--- + +## 2 · Key components + +| Component | Language / tech | Responsibility | +| ---------------------------- | --------------------- | ---------------------------------------------------- | +| **API Gateway** | ASP.NET Minimal API | Auth (JWT), quotas, request routing | +| **Scanner Core** | C# 12, Polly | Layer diffing, SBOM generation, vuln correlation | +| **Feedser (vulnerability ingest/merge/export service)** | C# source-gen workers | Consolidate NVD + regional CVE feeds into the canonical MongoDB store and drive JSON / Trivy DB exports | +| **Policy Engine** | OPA (Rego) | admission decisions, custom org rules | +| **Redis 7** | Key‑DB compatible | LRU cache, quota counters | +| **MongoDB 7** | WiredTiger | SBOM & findings storage | +| **Angular {{ angular }} UI** | RxJS, Tailwind | Dashboard, reports, admin UX | + +--- + +## 3 · Plug‑in system + +* Discovered once at start‑up from `/opt/stella/plugins/**`. +* Runs under Linux user `stella‑plugin` (UID 1001). +* Extension points: + + * `ISbomMutator` + * `IVulnerabilityProvider` + * `IResultSink` + * Policy files (`*.rego`) +* Each DLL is SHA‑256 hashed; digest embedded in the run report for provenance. + +Hot‑plugging is deferred until after v 1.0 for security review. + +--- + +## 4 · Data & control flow + +1. **Client** calls `/api/scan` with image reference. +2. **Gateway** enforces quota, forwards to **Scanner Core** via gRPC. +3. **Core**: + + * Queries Redis for cached SBOM. + * If miss → pulls layers, generates SBOM. + * Executes plug‑ins (mutators, additional scanners). +4. **Policy Engine** evaluates `scanResult` document. +5. **Findings** stored in MongoDB; WebSocket event notifies UI. +6. **ResultSink plug‑ins** export to Slack, Splunk, JSON file, etc. + +--- + +## 5 · Security hardening + +| Surface | Mitigation | +| ----------------- | ------------------------------------------------------------ | +| Container runtime | Distroless base, non‑root UID, seccomp + AppArmor | +| Plug‑in sandbox | Separate UID, SELinux profile, cgroup 1 CPU / 256 MiB | +| Supply chain | Cosign signatures, in‑toto SLSA Level 3 (target) | +| Secrets | `Docker secrets` or K8s `Secret` mounts; never hard‑coded | +| Quota abuse | Redis rate‑limit gates (see `30_QUOTA_ENFORCEMENT_FLOW1.md`) | + +--- + +## 6 · Build & release pipeline (TL;DR) + +* **Git commits** trigger CI → unit / integration / E2E tests. +* Successful merge to `main`: + + * Build `.NET {{ dotnet }}` trimmed self‑contained binary. + * `docker build --sbom=spdx-json`. + * Sign image and tarball with Cosign. + * Attach SBOM + provenance; push to registry and download portal. + +--- + +## 7 · Future extraction path + +Although the default deployment is a single container, each sub‑service can be +extracted: + +* Feedser → standalone cron pod. +* Policy Engine → side‑car (OPA) with gRPC contract. +* ResultSink → queue worker (RabbitMQ or Azure Service Bus). + +Interfaces are stable **as of v0.2 β**; extraction requires a recompilation +only, not a fork of the core. + +--- + +*Last updated {{ "now" | date: "%Y‑%m‑%d" }} – constants auto‑injected.* diff --git a/docs/60_POLICY_TEMPLATES.md b/docs/60_POLICY_TEMPLATES.md new file mode 100755 index 00000000..08eaca8d --- /dev/null +++ b/docs/60_POLICY_TEMPLATES.md @@ -0,0 +1,101 @@ +# Policy Templates — YAML & Rego Examples + +Stella Ops lets you enforce *pass / fail* rules in two ways: + +1. **YAML “quick policies”** — simple equality / inequality checks. +2. **OPA Rego modules** — full‑power logic for complex organisations. + +> **Precedence:** If the same image is subject to both a YAML rule *and* a Rego +> module, the **Rego result wins**. That is, `deny` in Rego overrides any +> `allow` in YAML. + +--- + +## 1 · YAML quick policy + +```yaml +# file: policies/root_user.yaml +version: 1 +id: root-user +description: Disallow images that run as root +severity: high + +rules: + - field: ".config.user" + operator: "equals" + value: "root" + deny_message: "Image runs as root — block." +```` + +Place the file under `/opt/stella/plugins/policies/`. + +--- + +## 2 · Rego example (deny on critical CVE) + +```rego +# file: policies/deny_critical.rego +package stella.policy + +default deny = [] + +deny[msg] { + some f + input.findings[f].severity == "critical" + msg := sprintf("Critical CVE %s – build blocked", [input.findings[f].id]) +} +``` + +*Input schema* — the Rego `input` document matches the public +`ScanResult` POCO (see SDK). Use the bundled JSON schema in +`share/schemas/scanresult.schema.json` for IDE autocompletion. + +--- + +## 3 · Pass‑through warnings (Rego) + +Return a `warn` array to surface non‑blocking messages in the UI: + +```rego +package stella.policy + +warn[msg] { + input.image.base == "ubuntu:16.04" + msg := "Image uses EOL Ubuntu 16.04 — please upgrade." +} +``` + +Warnings decrement the **quality score** but do *not* affect the CLI exit +code. + +--- + +## 4 · Testing policies locally + +```bash +# run policy evaluation without pushing to DB +stella scan alpine:3.20 --policy-only \ + --policies ./policies/ +``` + +The CLI prints `PASS`, `WARN` or `DENY` plus structured JSON. + +Unit‑test your Rego modules with the OPA binary: + +```bash +opa test policies/ +``` + +--- + +## 5 · Developer quick‑start (plug‑ins) + +Need logic beyond Rego? Implement a plug‑in via **C#/.NET {{ dotnet }}** and +the `StellaOps.SDK` NuGet: + +* Tutorial: [`dev/30_PLUGIN_DEV_GUIDE.md`](dev/30_PLUGIN_DEV_GUIDE.md) +* Quick reference: `/plugins/` + +--- + +*Last updated {{ "now" | date: "%Y‑%m‑%d" }} — constants auto‑injected.* diff --git a/docs/AGENTS.md b/docs/AGENTS.md new file mode 100644 index 00000000..56aa1d62 --- /dev/null +++ b/docs/AGENTS.md @@ -0,0 +1,19 @@ +# Docs & Enablement Guild + +## Mission +Produce and maintain offline-friendly documentation for StellaOps modules, covering architecture, configuration, operator workflows, and developer onboarding. + +## Scope Highlights +- Authority docs (`docs/dev/31_AUTHORITY_PLUGIN_DEVELOPER_GUIDE.md`, upcoming `docs/11_AUTHORITY.md`). +- Feedser quickstarts, CLI guides, Offline Kit manuals. +- Release notes and migration playbooks. + +## Operating Principles +- Keep guides deterministic and in sync with shipped configuration samples. +- Prefer tables/checklists for operator steps; flag security-sensitive actions. +- Update `docs/TASKS.md` whenever work items change status (TODO/DOING/REVIEW/DONE/BLOCKED). + +## Coordination +- Authority Core & Plugin teams for auth-related changes. +- Security Guild for threat-model outputs and mitigations. +- DevEx for tooling diagrams and documentation pipeline. diff --git a/docs/ARCHITECTURE_FEEDSER.md b/docs/ARCHITECTURE_FEEDSER.md new file mode 100644 index 00000000..9c11b8a6 --- /dev/null +++ b/docs/ARCHITECTURE_FEEDSER.md @@ -0,0 +1,190 @@ +# ARCHITECTURE.md — **StellaOps.Feedser** + +> **Goal**: Build a sovereign-ready, self-hostable **feed-merge service** that ingests authoritative vulnerability sources, normalizes and de-duplicates them into **MongoDB**, and exports **JSON** and **Trivy-compatible DB** artifacts. +> **Form factor**: Long-running **Web Service** with **REST APIs** (health, status, control) and an embedded **internal cron scheduler**. Controllable by StellaOps.Cli (# stella db ...) +> **No signing inside Feedser** (signing is a separate pipeline step). +> **Runtime SDK baseline**: .NET 10 Preview 7 (SDK 10.0.100-preview.7.25380.108) targeting `net10.0`, aligned with the deployed api.stella-ops.org service. +> **Four explicit stages**: +> +> 1. **Source Download** → raw documents. +> 2. **Parse & Normalize** → schema-validated DTOs enriched with canonical identifiers. +> 3. **Merge & Deduplicate** → precedence-aware canonical records persisted to MongoDB. +> 4. **Export** → JSON or TrivyDB (full or delta), then (externally) sign/publish. + +--- + +## 1) Naming & Solution Layout + +**Source connectors** namespace prefix: `StellaOps.Feedser.Source.*` +**Exporters**: + +* `StellaOps.Feedser.Exporter.Json` +* `StellaOps.Feedser.Exporter.TrivyDb` + +**Projects** (`/src`): + +``` +StellaOps.Feedser.WebService/ # ASP.NET Core (Minimal API, net10.0 preview) WebService + embedded scheduler +StellaOps.Feedser.Core/ # Domain models, pipelines, merge/dedupe engine, jobs orchestration +StellaOps.Feedser.Models/ # Canonical POCOs, JSON Schemas, enums +StellaOps.Feedser.Storage.Mongo/ # Mongo repositories, GridFS access, indexes, resume "flags" +StellaOps.Feedser.Source.Common/ # HTTP clients, rate-limiters, schema validators, parsers utils +StellaOps.Feedser.Source.Cve/ +StellaOps.Feedser.Source.Nvd/ +StellaOps.Feedser.Source.Ghsa/ +StellaOps.Feedser.Source.Osv/ +StellaOps.Feedser.Source.Jvn/ +StellaOps.Feedser.Source.CertCc/ +StellaOps.Feedser.Source.Kev/ +StellaOps.Feedser.Source.Kisa/ +StellaOps.Feedser.Source.CertIn/ +StellaOps.Feedser.Source.CertFr/ +StellaOps.Feedser.Source.CertBund/ +StellaOps.Feedser.Source.Acsc/ +StellaOps.Feedser.Source.Cccs/ +StellaOps.Feedser.Source.Ru.Bdu/ # HTML→schema with LLM fallback (gated) +StellaOps.Feedser.Source.Ru.Nkcki/ # PDF/HTML bulletins → structured +StellaOps.Feedser.Source.Vndr.Msrc/ +StellaOps.Feedser.Source.Vndr.Cisco/ +StellaOps.Feedser.Source.Vndr.Oracle/ +StellaOps.Feedser.Source.Vndr.Adobe/ # APSB ingest; emits vendor RangePrimitives with adobe.track/platform/priority telemetry + fixed-status provenance. +StellaOps.Feedser.Source.Vndr.Apple/ +StellaOps.Feedser.Source.Vndr.Chromium/ +StellaOps.Feedser.Source.Vndr.Vmware/ +StellaOps.Feedser.Source.Distro.RedHat/ +StellaOps.Feedser.Source.Distro.Debian/ # Fetches DSA list + detail HTML, emits EVR RangePrimitives with per-release provenance and telemetry. +StellaOps.Feedser.Source.Distro.Ubuntu/ # Ubuntu Security Notices connector (JSON index → EVR ranges with ubuntu.pocket telemetry). +StellaOps.Feedser.Source.Distro.Suse/ # CSAF fetch pipeline emitting NEVRA RangePrimitives with suse.status vendor telemetry. +StellaOps.Feedser.Source.Ics.Cisa/ +StellaOps.Feedser.Source.Ics.Kaspersky/ +StellaOps.Feedser.Normalization/ # Canonical mappers, validators, version-range normalization +StellaOps.Feedser.Merge/ # Identity graph, precedence, deterministic merge +StellaOps.Feedser.Exporter.Json/ +StellaOps.Feedser.Exporter.TrivyDb/ +StellaOps.Feedser..Tests/ # Component-scoped unit/integration suites (Core, Storage.Mongo, Source.*, Exporter.*, WebService, etc.) +``` + +--- + +## 2) Runtime Shape + +**Process**: single service (`StellaOps.Feedser.WebService`) + +* `Program.cs`: top-level entry using **Generic Host**, **DI**, **Options** binding from `appsettings.json` + environment + optional `feedser.yaml`. +* Built-in **scheduler** (cron-like) + **job manager** with **distributed locks** in Mongo to prevent overlaps, enforce timeouts, allow cancel/kill. +* **REST APIs** for health/readiness/progress/trigger/kill/status. + +**Key NuGet concepts** (indicative): `MongoDB.Driver`, `Polly` (retry/backoff), `System.Threading.Channels`, `Microsoft.Extensions.Http`, `Microsoft.Extensions.Hosting`, `Serilog`, `OpenTelemetry`. + +--- + +## 3) Data Storage — **MongoDB** (single source of truth) + +**Database**: `feedser` +**Write concern**: `majority` for merge/export state, `acknowledged` for raw docs. +**Collections** (with “flags”/resume points): + +* `source` + * `_id`, `name`, `type`, `baseUrl`, `auth`, `notes`. +* `source_state` + * Keys: `sourceName` (unique), `enabled`, `cursor`, `lastSuccess`, `failCount`, `backoffUntil`, `paceOverrides`, `paused`. + * Drives incremental fetch/parse/map resume and operator pause/pace controls. +* `document` + * `_id`, `sourceName`, `uri`, `fetchedAt`, `sha256`, `contentType`, `status`, `metadata`, `gridFsId`, `etag`, `lastModified`. + * Index `{sourceName:1, uri:1}` unique; optional TTL for superseded versions. +* `dto` + * `_id`, `sourceName`, `documentId`, `schemaVer`, `payload` (BSON), `validatedAt`. + * Index `{sourceName:1, documentId:1}`. +* `advisory` + * `_id`, `advisoryKey`, `title`, `summary`, `lang`, `published`, `modified`, `severity`, `exploitKnown`. + * Unique `{advisoryKey:1}` plus indexes on `modified` and `published`. +* `alias` + * `advisoryId`, `scheme`, `value` with index `{scheme:1, value:1}`. +* `affected` + * `advisoryId`, `platform`, `name`, `versionRange`, `cpe`, `purl`, `fixedBy`, `introducedVersion`. + * Index `{platform:1, name:1}`, `{advisoryId:1}`. +* `reference` + * `advisoryId`, `url`, `kind`, `sourceTag` (e.g., advisory/patch/kb). +* Flags collections: `kev_flag`, `ru_flags`, `jp_flags`, `psirt_flags` keyed by `advisoryId`. +* `merge_event` + * `_id`, `advisoryKey`, `beforeHash`, `afterHash`, `mergedAt`, `inputs` (document ids). +* `export_state` + * `_id` (`json`/`trivydb`), `baseExportId`, `baseDigest`, `lastFullDigest`, `lastDeltaDigest`, `exportCursor`, `targetRepo`, `exporterVersion`. +* `locks` + * `_id` (`jobKey`), `holder`, `acquiredAt`, `heartbeatAt`, `leaseMs`, `ttlAt` (TTL index cleans dead locks). +* `jobs` + * `_id`, `type`, `args`, `state`, `startedAt`, `endedAt`, `error`, `owner`, `heartbeatAt`, `timeoutMs`. + +**GridFS buckets**: `fs.documents` for raw large payloads; referenced by `document.gridFsId`. + +--- + +## 4) Job & Scheduler Model + +* Scheduler stores cron expressions per source/exporter in config; persists next-run pointers in Mongo. +* Jobs acquire locks (`locks` collection) to ensure singleton execution per source/exporter. +* Supports manual triggers via API endpoints (`POST /jobs/{type}`) and pause/resume toggles per source. + +--- + +## 5) Connector Contracts + +Connectors implement: + +```csharp +public interface IFeedConnector { + string SourceName { get; } + Task FetchAsync(IServiceProvider sp, CancellationToken ct); + Task ParseAsync(IServiceProvider sp, CancellationToken ct); + Task MapAsync(IServiceProvider sp, CancellationToken ct); +} +``` + +* Fetch populates `document` rows respecting rate limits, conditional GET, and `source_state.cursor`. +* Parse validates schema (JSON Schema, XSD) and writes sanitized DTO payloads. +* Map produces canonical advisory rows + provenance entries; must be idempotent. +* Base helpers in `StellaOps.Feedser.Source.Common` provide HTTP clients, retry policies, and watermark utilities. + +--- + +## 6) Merge & Normalization + +* Canonical model stored in `StellaOps.Feedser.Models` with serialization contracts used by storage/export layers. +* `StellaOps.Feedser.Normalization` handles NEVRA/EVR/PURL range parsing, CVSS normalization, localization. +* `StellaOps.Feedser.Merge` builds alias graphs keyed by CVE first, then falls back to vendor/regional IDs. +* Precedence rules: PSIRT/OVAL overrides generic ranges; KEV only toggles exploitation; regional feeds enrich severity but don’t override vendor truth. +* Determinism enforced via canonical JSON hashing logged in `merge_event`. + +--- + +## 7) Exporters + +* JSON exporter mirrors `aquasecurity/vuln-list` layout with deterministic ordering and reproducible timestamps. +* Trivy DB exporter shells out to `trivy-db build`, produces Bolt archives, and reuses unchanged blobs from the last full baseline when running in delta mode. The exporter annotates `metadata.json` with `mode`, `baseExportId`, `baseManifestDigest`, `resetBaseline`, and `delta.changedFiles[]`/`delta.removedPaths[]`, and honours `publishFull` / `publishDelta` (ORAS) plus `includeFull` / `includeDelta` (offline bundle) toggles. +* `StellaOps.Feedser.Storage.Mongo` provides cursors for delta exports based on `export_state.exportCursor` and the persisted per-file manifest (`export_state.files`). +* Export jobs produce OCI tarballs (layer media type `application/vnd.aquasec.trivy.db.layer.v1.tar+gzip`) and optionally push via ORAS; `metadata.json` accompanies each layout so mirrors can decide between full refreshes and deltas. + +--- + +## 8) Observability + +* Serilog structured logging with enrichment fields (`source`, `uri`, `stage`, `durationMs`). +* OpenTelemetry traces around fetch/parse/map/export; metrics for rate limit hits, schema failures, dedupe ratios, package size. Connector HTTP metrics are emitted via the shared `feedser.source.http.*` instruments tagged with `feedser.source=` so per-source dashboards slice on that label instead of bespoke metric names. +* Prometheus scraping endpoint served by WebService. + +--- + +## 9) Security Considerations + +* Offline-first: connectors only reach allowlisted hosts. +* BDU LLM fallback gated by config flag; logs audit trail with confidence score. +* No secrets written to logs; secrets loaded via environment or mounted files. +* Signing handled outside Feedser pipeline. + +--- + +## 10) Deployment Notes + +* Default storage MongoDB; for air-gapped, bundle Mongo image + seeded data backup. +* Horizontal scale achieved via multiple web service instances sharing Mongo locks. +* Provide `feedser.yaml` template describing sources, rate limits, and export settings. diff --git a/docs/README.md b/docs/README.md new file mode 100755 index 00000000..c1a12ece --- /dev/null +++ b/docs/README.md @@ -0,0 +1,67 @@ +# Stella Ops + +> **Self‑hosted, SBOM‑first DevSecOps platform – offline‑friendly, AGPL‑3.0, free up to {{ quota_token }} scans per UTC day (soft delay only, never blocks).** + +Stella Ops lets you discover container vulnerabilities in **< 5 s** without sending a single byte outside your network. +Everything here is open‑source and versioned — when you check out a git tag, the docs match the code you are running. + +--- + +## 🚀 Start here (first 60 minutes) + +| Step | What you will learn | Doc | +|------|--------------------|-----| +| 1 ️⃣ | 90‑second elevator pitch & pillars | **[What Is Stella Ops?](01_WHAT_IS_IT.md)** | +| 2 ️⃣ | Pain points it solves | **[Why Does It Exist?](02_WHY.md)** | +| 3 ️⃣ | Install & run a scan in 10 min | **[Install Guide](21_INSTALL_GUIDE.md)** | +| 4 ️⃣ | Components & data‑flow | **[High‑Level Architecture](07_HIGH_LEVEL_ARCHITECTURE.md)** | +| 5 ️⃣ | Integrate the CLI / REST API | **[API & CLI Reference](09_API_CLI_REFERENCE.md)** | +| 6 ️⃣ | Vocabulary used throughout the docs | **[Glossary](14_GLOSSARY_OF_TERMS.md)** | + +--- + +## 📚 Complete Table of Contents + +
+Click to expand the full docs index + +### Overview +- **01 – [What Is Stella Ops?](01_WHAT_IS_IT.md)** +- **02 – [Why Does It Exist?](02_WHY.md)** +- **03 – [Vision & Road‑map](03_VISION.md)** +- **04 – [Feature Matrix](04_FEATURE_MATRIX.md)** + +### Reference & concepts +- **05 – [System Requirements Specification](05_SYSTEM_REQUIREMENTS_SPEC.md)** +- **07 – [High‑Level Architecture](40_ARCHITECTURE_OVERVIEW.md)** +- **08 – Module Specifications** + - [README](08_MODULE_SPECIFICATIONS/README.md) + - [`backend_api.md`](08_MODULE_SPECIFICATIONS/backend_api.md) + - [`zastava_scanner.md`](08_MODULE_SPECIFICATIONS/zastava_scanner.md) + - [`registry_scanner.md`](08_MODULE_SPECIFICATIONS/registry_scanner.md) + - [`nightly_scheduler.md`](08_MODULE_SPECIFICATIONS/nightly_scheduler.md) +- **09 – [API & CLI Reference](09_API_CLI_REFERENCE.md)** +- **10 – [Plug‑in SDK Guide](10_PLUGIN_SDK_GUIDE.md)** +- **11 – [Data Schemas](11_DATA_SCHEMAS.md)** +- **12 – [Performance Workbook](12_PERFORMANCE_WORKBOOK.md)** +- **13 – [Release‑Engineering Playbook](13_RELEASE_ENGINEERING_PLAYBOOK.md)** + +### User & operator guides +- **14 – [Glossary](14_GLOSSARY_OF_TERMS.md)** +- **15 – [UI Guide](15_UI_GUIDE.md)** +- **17 – [Security Hardening Guide](17_SECURITY_HARDENING_GUIDE.md)** +- **18 – [Coding Standards](18_CODING_STANDARDS.md)** +- **19 – [Test‑Suite Overview](19_TEST_SUITE_OVERVIEW.md)** +- **21 – [Install Guide](21_INSTALL_GUIDE.md)** +- **22 – [CI/CD Recipes Library](ci/20_CI_RECIPES.md)** +- **23 – [FAQ](23_FAQ_MATRIX.md)** +- **24 – [Offline Update Kit Admin Guide](24_OUK_ADMIN_GUIDE.md)** + +### Legal & licence +- **29 – [Legal & Quota FAQ](29_LEGAL_FAQ_QUOTA.md)** + +
+ +--- + +© 2025 Stella Ops contributors – licensed AGPL‑3.0‑or‑later diff --git a/docs/TASKS.md b/docs/TASKS.md new file mode 100644 index 00000000..9ac95b47 --- /dev/null +++ b/docs/TASKS.md @@ -0,0 +1,12 @@ +# Docs Guild Task Board (UTC 2025-10-10) + +| ID | Status | Owner(s) | Depends on | Description | Exit Criteria | +|----|--------|----------|------------|-------------|---------------| +| DOC4.AUTH-PDG | REVIEW | Docs Guild, Plugin Team | PLG6.DOC | Copy-edit `docs/dev/31_AUTHORITY_PLUGIN_DEVELOPER_GUIDE.md`, export lifecycle diagram, add LDAP RFC cross-link. | ✅ PR merged with polish; ✅ Diagram committed; ✅ Slack handoff posted. | +| DOC1.AUTH | TODO | Docs Guild, Authority Core | CORE5B.DOC | Draft `docs/11_AUTHORITY.md` covering architecture, configuration, bootstrap flows. | ✅ Architecture + config sections approved by Core; ✅ Samples reference latest options; ✅ Offline note added. | +| DOC3.Feedser-Authority | DOING (2025-10-10) | Docs Guild, DevEx | FSR4 | Polish operator/runbook sections (DOC3/DOC5) to document Feedser authority rollout, bypass logging, and enforcement checklist. | ✅ DOC3/DOC5 updated; ✅ enforcement deadline highlighted; ✅ Docs guild sign-off. | +| DOC5.Feedser-Runbook | TODO | Docs Guild | DOC3.Feedser-Authority | Produce dedicated Feedser authority audit runbook covering log fields, monitoring recommendations, and troubleshooting steps. | ✅ Runbook published; ✅ linked from DOC3/DOC5; ✅ alerting guidance included. | +| FEEDDOCS-DOCS-05-001 | DONE (2025-10-11) | Docs Guild | FEEDMERGE-ENGINE-04-001, FEEDMERGE-ENGINE-04-002 | Publish Feedser conflict resolution runbook covering precedence workflow, merge-event auditing, and Sprint 3 metrics. | ✅ `docs/ops/feedser-conflict-resolution.md` committed; ✅ metrics/log tables align with latest merge code; ✅ Ops alert guidance handed to Feedser team. | +| FEEDDOCS-DOCS-05-002 | TODO | Docs Guild, Feedser Ops | FEEDDOCS-DOCS-05-001 | Capture ops sign-off: circulate conflict runbook, tune alert thresholds, and document rollout decisions in change log. | ✅ Ops review recorded; ✅ alert thresholds finalised; ✅ change-log entry linked from runbook. | + +> Update statuses (TODO/DOING/REVIEW/DONE/BLOCKED) as progress changes. Keep guides in sync with configuration samples under `etc/`. diff --git a/docs/_includes/CONSTANTS.md b/docs/_includes/CONSTANTS.md new file mode 100755 index 00000000..efde601b --- /dev/null +++ b/docs/_includes/CONSTANTS.md @@ -0,0 +1,18 @@ +### `docs/_includes/CONSTANTS.md` + +```yaml +--- +# ───────────────────────────────────────────────────────────────────────────── +# Shared constants for both the technical docs (Markdown) and the marketing +# site (Nunjucks). Eleventy injects these variables into every template. +# Never hard‑code the values elsewhere — lint‑ci will block the merge. +# ───────────────────────────────────────────────────────────────────────────── + +dotnet: "10 LTS" # Runs on .NET 10 (LTS channel) +angular: "20" # Front‑end framework major +quota_anon: 33 # Anonymous daily scans +quota_token: 333 # Daily scans with free JWT +slowdown: "5–60 s" # Delay window after exceeding quota + +# Add new keys here; update the docs linter pattern in .gitlab-ci.yml. +--- \ No newline at end of file diff --git a/docs/ci/20_CI_RECIPES.md b/docs/ci/20_CI_RECIPES.md new file mode 100755 index 00000000..4ad86464 --- /dev/null +++ b/docs/ci/20_CI_RECIPES.md @@ -0,0 +1,258 @@ +# Stella Ops CI Recipes — (2025‑08‑04) + +## 0 · Key variables (export these once) + +| Variable | Meaning | Typical value | +| ------------- | --------------------------------------------------------------------------------------------------------------------------------- | ---------------------------------------------------- | +| `STELLA_URL` | Host that: ① stores the **CLI** & **SBOM‑builder** images under `/registry` **and** ② receives API calls at `https://$STELLA_URL` | `stella-ops.ci.acme.example` | +| `DOCKER_HOST` | How containers reach your Docker daemon (because we no longer mount `/var/run/docker.sock`) | `tcp://docker:2375` | +| `WORKSPACE` | Directory where the pipeline stores artefacts (SBOM file) | `$(pwd)` | +| `IMAGE` | The image you are building & scanning | `acme/backend:sha-${COMMIT_SHA}` | +| `SBOM_FILE` | Immutable SBOM name – `‑YYYYMMDDThhmmssZ.sbom.json` | `acme_backend_sha‑abc123‑20250804T153050Z.sbom.json` | + +```bash +export STELLA_URL="stella-ops.ci.acme.example" +export DOCKER_HOST="tcp://docker:2375" # Jenkins/Circle often expose it like this +export WORKSPACE="$(pwd)" +export IMAGE="acme/backend:sha-${COMMIT_SHA}" +export SBOM_FILE="$(echo "${IMAGE}" | tr '/:+' '__')-$(date -u +%Y%m%dT%H%M%SZ).sbom.json" +``` + +--- + +## 1 · SBOM creation strategies + +### Option A – **Buildx attested SBOM** (preferred if you can use BuildKit) + +You pass **two build args** so the Dockerfile can run the builder and copy the result out of the build context. + +```bash +docker buildx build \ + --build-arg STELLA_SBOM_BUILDER="$STELLA_URL/registry/stella-sbom-builder:latest" \ + --provenance=true --sbom=true \ + --build-arg SBOM_FILE="$SBOM_FILE" \ + -t "$IMAGE" . +``` + +**If you **cannot** use Buildx, use Option B below.** The older “run a builder stage inside the Dockerfile” pattern is unreliable for producing an SBOM of the final image. + +```Dockerfile + +ARG STELLA_SBOM_BUILDER +ARG SBOM_FILE + +FROM $STELLA_SBOM_BUILDER as sbom +ARG IMAGE +ARG SBOM_FILE +RUN $STELLA_SBOM_BUILDER build --image $IMAGE --output /out/$SBOM_FILE + +# ---- actual build stages … ---- +FROM alpine:3.20 +COPY --from=sbom /out/$SBOM_FILE / # (optional) keep or discard + +# (rest of your Dockerfile) +``` + +### Option B – **External builder step** (works everywhere; recommended baseline if Buildx isn’t available) + +*(keep this block if your pipeline already has an image‑build step that you can’t modify)* + +```bash +docker run --rm \ + -e DOCKER_HOST="$DOCKER_HOST" \ # let builder reach the daemon remotely + -v "$WORKSPACE:/workspace" \ # place SBOM beside the source code + "$STELLA_URL/registry/stella-sbom-builder:latest" \ + build --image "$IMAGE" --output "/workspace/${SBOM_FILE}" +``` + +--- + +## 2 · Scan the image & upload results + +```bash +docker run --rm \ + -e DOCKER_HOST="$DOCKER_HOST" \ # remote‑daemon pointer + -v "$WORKSPACE/${SBOM_FILE}:/${SBOM_FILE}:ro" \ # mount SBOM under same name at container root + -e STELLA_OPS_URL="https://${STELLA_URL}" \ # where the CLI posts findings + "$STELLA_URL/registry/stella-cli:latest" \ + scan --sbom "/${SBOM_FILE}" "$IMAGE" +``` + +The CLI returns **exit 0** if policies pass, **>0** if blocked — perfect for failing the job. + +--- + +## 3 · CI templates + +Below are minimal, cut‑and‑paste snippets. +**Feel free to delete Option B** if you adopt Option A. + +### 3.1 Jenkins (Declarative Pipeline) + +```groovy +pipeline { + agent { docker { image 'docker:25' args '--privileged' } } // gives us /usr/bin/docker + environment { + STELLA_URL = 'stella-ops.ci.acme.example' + DOCKER_HOST = 'tcp://docker:2375' + IMAGE = "acme/backend:${env.BUILD_NUMBER}" + SBOM_FILE = "acme_backend_${env.BUILD_NUMBER}-${new Date().format('yyyyMMdd\'T\'HHmmss\'Z\'', TimeZone.getTimeZone('UTC'))}.sbom.json" + } + stages { + stage('Build image + SBOM (Option A)') { + steps { + sh ''' + docker build \ + --build-arg STELLA_SBOM_BUILDER="$STELLA_URL/registry/stella-sbom-builder:latest" \ + --build-arg SBOM_FILE="$SBOM_FILE" \ + -t "$IMAGE" . + ''' + } + } + /* ---------- Option B fallback (when you must keep the existing build step as‑is) ---------- + stage('SBOM builder (Option B)') { + steps { + sh ''' + docker run --rm -e DOCKER_HOST="$DOCKER_HOST" \ + -v "$WORKSPACE:/workspace" \ + "$STELLA_URL/registry/stella-sbom-builder:latest" \ + build --image "$IMAGE" --output "/workspace/${SBOM_FILE}" + ''' + } + } + ------------------------------------------------------------------------------------------ */ + stage('Scan & upload') { + steps { + sh ''' + docker run --rm -e DOCKER_HOST="$DOCKER_HOST" \ + -v "$WORKSPACE/${SBOM_FILE}:/${SBOM_FILE}:ro" \ + -e STELLA_OPS_URL="https://$STELLA_URL" \ + "$STELLA_URL/registry/stella-cli:latest" \ + scan --sbom "/${SBOM_FILE}" "$IMAGE" + ''' + } + } + } +} +``` + +--- + +### 3.2 CircleCI `.circleci/config.yml` + +```yaml +version: 2.1 +jobs: + stella_scan: + docker: + - image: cimg/base:stable # baremetal image with Docker CLI + environment: + STELLA_URL: stella-ops.ci.acme.example + DOCKER_HOST: tcp://docker:2375 # Circle’s “remote Docker” socket + steps: + - checkout + + - run: + name: Compute vars + command: | + echo 'export IMAGE="acme/backend:${CIRCLE_SHA1}"' >> $BASH_ENV + echo 'export SBOM_FILE="$(echo acme/backend:${CIRCLE_SHA1} | tr "/:+" "__")-$(date -u +%Y%m%dT%H%M%SZ).sbom.json"' >> $BASH_ENV + - run: + name: Build image + SBOM (Option A) + command: | + docker build \ + --build-arg STELLA_SBOM_BUILDER="$STELLA_URL/registry/stella-sbom-builder:latest" \ + --build-arg SBOM_FILE="$SBOM_FILE" \ + -t "$IMAGE" . + # --- Option B fallback (when you must keep the existing build step as‑is) --- + #- run: + # name: SBOM builder (Option B) + # command: | + # docker run --rm -e DOCKER_HOST="$DOCKER_HOST" \ + # -v "$PWD:/workspace" \ + # "$STELLA_URL/registry/stella-sbom-builder:latest" \ + # build --image "$IMAGE" --output "/workspace/${SBOM_FILE}" + - run: + name: Scan + command: | + docker run --rm -e DOCKER_HOST="$DOCKER_HOST" \ + -v "$PWD/${SBOM_FILE}:/${SBOM_FILE}:ro" \ + -e STELLA_OPS_URL="https://$STELLA_URL" \ + "$STELLA_URL/registry/stella-cli:latest" \ + scan --sbom "/${SBOM_FILE}" "$IMAGE" +workflows: + stella: + jobs: [stella_scan] +``` + +--- + +### 3.3 Gitea Actions `.gitea/workflows/stella.yml` + +*(Gitea 1.22+ ships native Actions compatible with GitHub syntax)* + +```yaml +name: Stella Scan +on: [push] + +jobs: + stella: + runs-on: ubuntu-latest + env: + STELLA_URL: ${{ secrets.STELLA_URL }} + DOCKER_HOST: tcp://docker:2375 # provided by the docker:dind service + services: + docker: + image: docker:dind + options: >- + --privileged + steps: + - uses: actions/checkout@v4 + + - name: Compute vars + id: vars + run: | + echo "IMAGE=ghcr.io/${{ gitea.repository }}:${{ gitea.sha }}" >> $GITEA_OUTPUT + echo "SBOM_FILE=$(echo ghcr.io/${{ gitea.repository }}:${{ gitea.sha }} | tr '/:+' '__')-$(date -u +%Y%m%dT%H%M%SZ).sbom.json" >> $GITEA_OUTPUT + + - name: Build image + SBOM (Option A) + run: | + docker build \ + --build-arg STELLA_SBOM_BUILDER="${STELLA_URL}/registry/stella-sbom-builder:latest" \ + --build-arg SBOM_FILE="${{ steps.vars.outputs.SBOM_FILE }}" \ + -t "${{ steps.vars.outputs.IMAGE }}" . + + # --- Option B fallback (when you must keep the existing build step as‑is) --- + #- name: SBOM builder (Option B) + # run: | + # docker run --rm -e DOCKER_HOST="$DOCKER_HOST" \ + # -v "$(pwd):/workspace" \ + # "${STELLA_URL}/registry/stella-sbom-builder:latest" \ + # build --image "${{ steps.vars.outputs.IMAGE }}" --output "/workspace/${{ steps.vars.outputs.SBOM_FILE }}" + + - name: Scan + run: | + docker run --rm -e DOCKER_HOST="$DOCKER_HOST" \ + -v "$(pwd)/${{ steps.vars.outputs.SBOM_FILE }}:/${{ steps.vars.outputs.SBOM_FILE }}:ro" \ + -e STELLA_OPS_URL="https://${STELLA_URL}" \ + "${STELLA_URL}/registry/stella-cli:latest" \ + scan --sbom "/${{ steps.vars.outputs.SBOM_FILE }}" "${{ steps.vars.outputs.IMAGE }}" +``` + +--- + +## 4 · Troubleshooting cheat‑sheet + +| Symptom | Root cause | First things to try | +| ------------------------------------- | --------------------------- | --------------------------------------------------------------- | +| `no such host $STELLA_URL` | DNS typo or VPN outage | `ping $STELLA_URL` from runner | +| `connection refused` when CLI uploads | Port 443 blocked | open firewall / check ingress | +| `failed to stat /.json` | SBOM wasn’t produced | Did Option A actually run builder? If not, enable Option B | +| `registry unauthorized` | Runner lacks registry creds | `docker login $STELLA_URL/registry` (store creds in CI secrets) | +| Non‑zero scan exit | Blocking vuln/licence | Open project in Ops UI → triage or waive | + +--- + +### Change log + +* **2025‑08‑04** – Variable clean‑up, removed Docker‑socket & cache mounts, added Jenkins / CircleCI / Gitea examples, clarified Option B comment. diff --git a/docs/cli/20_REFERENCE.md b/docs/cli/20_REFERENCE.md new file mode 100755 index 00000000..941aa692 --- /dev/null +++ b/docs/cli/20_REFERENCE.md @@ -0,0 +1,8 @@ +# CLI Reference (`stella --help`) + +> **Auto‑generated file — do not edit manually.** +> On every tagged release the CI pipeline runs +> `stella --help --markdown > docs/cli/20_REFERENCE.md` +> ensuring this document always matches the shipped binary. + +*(The reference will appear after the first public α release.)* diff --git a/docs/dev/30_PLUGIN_DEV_GUIDE.md b/docs/dev/30_PLUGIN_DEV_GUIDE.md new file mode 100755 index 00000000..12e21827 --- /dev/null +++ b/docs/dev/30_PLUGIN_DEV_GUIDE.md @@ -0,0 +1,146 @@ +# Writing Plug‑ins for Stella Ops SDK *Preview 3* + +> **SDK status:** *Preview 3* is compatible with the **v0.1 α** runtime. +> Interfaces freeze at **v0.2 β**; binary‑breaking changes are still possible +> until then. + +| SDK NuGet | Runtime compat | Notes | +|-----------|---------------|-------| +| `StellaOps.SDK 0.2.0-preview3` | `stella-ops >= 0.1.0-alpha` | Current preview | +| `StellaOps.SDK 0.2.x‑beta` | v0.2 β (Q1 2026) | Interface **freeze** | +| `StellaOps.SDK 1.0.0` | v1.0 GA (Q4 2026) | Semantic Ver from here | + +--- + +## 0 · Extension points + +| Area | Interface / format | Example | +|------|--------------------|---------| +| SBOM mutator | `ISbomMutator` | Inject SPDX licences | +| Additional scanner | `IVulnerabilityProvider` | Rust Crates ecosystem | +| Policy engine | **OPA Rego** file | Custom pass/fail rule | +| Result exporter | `IResultSink` | Slack webhook notifier | + +*Hot‑plugging (live reload) is **post‑1.0**; modules are discovered once +during service start‑up.* + +--- + +## 1 · Five‑minute quick‑start (C# /.NET {{ dotnet }}) + +```bash +dotnet new classlib -n SlackExporter +cd SlackExporter +dotnet add package StellaOps.SDK --version 0.2.0-preview3 +```` + +```csharp +using System.Net.Http.Json; +using StellaOps.Plugin; + +public sealed class SlackSink : IResultSink +{ + private readonly string _webhook = + Environment.GetEnvironmentVariable("SLACK_WEBHOOK") + ?? throw new InvalidOperationException("Missing SLACK_WEBHOOK"); + + public string Name => "Slack Notifier"; + + public async Task ExportAsync(ScanResult result, CancellationToken ct) + { + var payload = new + { + text = $":rotating_light: *{result.Image}* " + + $"→ {result.Findings.Count} findings (max {result.MaxSeverity})" + }; + + using var client = new HttpClient(); + await client.PostAsJsonAsync(_webhook, payload, ct); + } +} +``` + +```bash +dotnet publish -c Release -o out +sudo mkdir -p /opt/stella/plugins/Slack +sudo cp out/SlackExporter.dll /opt/stella/plugins/Slack/ +sudo systemctl restart stella-ops +``` + +Start‑up log: + +``` +[PluginLoader] Loaded 1 plug‑in: + • Slack Notifier +``` + +--- + +## 2 · Packaging rules + +| Item | Rule | +| ------ | ----------------------------------------- | +| Folder | `/opt/stella/plugins//` | +| DLLs | Your plug‑in + non‑GAC deps | +| Config | Env‑vars or `settings.yaml` | +| SBOM | Optional `addon.spdx.json` for provenance | + +--- + +## 3 · Security sandbox + +* Runs as Linux user **`stella‑plugin` (UID 1001)**. +* SELinux/AppArmor profile blocks inbound traffic; outbound :80/443 only. +* cgroup default: **1 CPU / 256 MiB** (adjustable). +* SHA‑256 of every DLL is embedded in the run report. + +--- + +## 4 · Debugging + +| Technique | Command | +| ----------------- | ---------------------------------- | +| Verbose core log | `STELLA_LOG=debug` | +| Per‑plug‑in log | Inject `ILogger` | +| Dry‑run (no fail) | `--plugin-mode warn` | +| Hot reload | *Not supported* (planned post‑1.0) | + +Logs: `/var/log/stella-ops/plugins/YYYY‑MM‑DD.log`. + +--- + +## 5 · Interface reference (Preview 3) + +```csharp +namespace StellaOps.Plugin +{ + public interface ISbomMutator + { + string Name { get; } + Task MutateAsync( + SoftwareBillOfMaterials sbom, + CancellationToken ct = default); + } + + public interface IVulnerabilityProvider + { + string Ecosystem { get; } + Task> QueryAsync( + PackageReference p, CancellationToken ct = default); + } + + public interface IResultSink + { + string Name { get; } + Task ExportAsync( + ScanResult result, CancellationToken ct = default); + } +} +``` + +Full POCO docs: [https://git.stella-ops.org/stella-ops/sdk/-/tree/main/docs/api](https://git.stella-ops.org/stella-ops/sdk/-/tree/main/docs/api). + +--- + +*Last updated {{ "now" | date: "%Y‑%m‑%d" }} – constants auto‑injected.* + diff --git a/docs/dev/31_AUTHORITY_PLUGIN_DEVELOPER_GUIDE.md b/docs/dev/31_AUTHORITY_PLUGIN_DEVELOPER_GUIDE.md new file mode 100644 index 00000000..36a4e76b --- /dev/null +++ b/docs/dev/31_AUTHORITY_PLUGIN_DEVELOPER_GUIDE.md @@ -0,0 +1,157 @@ +# Authority Plug-in Developer Guide + +> **Status:** Ready for Docs/DOC4 editorial review as of 2025-10-10. Content aligns with PLG6 acceptance criteria and references stable Authority primitives. + +## 1. Overview +Authority plug-ins extend the **StellaOps Authority** service with custom identity providers, credential stores, and client-management logic. Unlike Feedser plug-ins (which ingest or export advisories), Authority plug-ins participate directly in authentication flows: + +- **Use cases:** integrate corporate directories (LDAP/AD), delegate to external IDPs, enforce bespoke password/lockout policies, or add client provisioning automation. +- **Constraints:** plug-ins load only during service start (no hot-reload), must function without outbound internet access, and must emit deterministic results for identical configuration and input data. +- **Ship targets:** target the same .NET 10 preview as the host, honour offline-first requirements, and provide clear diagnostics so operators can triage issues from `/ready`. + +## 2. Architecture Snapshot +Authority hosts follow a deterministic plug-in lifecycle. The flow below can be rendered as a sequence diagram in the final authored documentation, but all touchpoints are described here for offline viewers: + +1. **Configuration load** – `AuthorityPluginConfigurationLoader` resolves YAML manifests under `etc/authority.plugins/`. +2. **Assembly discovery** – the shared `PluginHost` scans `PluginBinaries/Authority` for `StellaOps.Authority.Plugin.*.dll` assemblies. +3. **Registrar execution** – each assembly is searched for `IAuthorityPluginRegistrar` implementations. Registrars bind options, register services, and optionally queue bootstrap tasks. +4. **Runtime** – the host resolves `IIdentityProviderPlugin` instances, uses capability metadata to decide which OAuth grants to expose, and invokes health checks for readiness endpoints. + +**Data persistence primer:** the standard Mongo-backed plugin stores users in collections named `authority_users_` and lockout metadata in embedded documents. Additional plugins must document their storage layout and provide deterministic collection naming to honour the Offline Kit replication process. + +## 3. Capability Metadata +Capability flags let the host reason about what your plug-in supports: + +- Declare capabilities in your descriptor using the string constants from `AuthorityPluginCapabilities` (`password`, `mfa`, `clientProvisioning`, `bootstrap`). The configuration loader now validates these tokens and rejects unknown values at startup. +- `AuthorityIdentityProviderCapabilities.FromCapabilities` projects those strings into strongly typed booleans (`SupportsPassword`, etc.). Authority Core will use these flags when wiring flows such as the password grant. Built-in plugins (e.g., Standard) will fail fast or force-enable required capabilities if the descriptor is misconfigured, so keep manifests accurate. +- Typical configuration (`etc/authority.plugins/standard.yaml`): + ```yaml + plugins: + descriptors: + standard: + assemblyName: "StellaOps.Authority.Plugin.Standard" + capabilities: + - password + - bootstrap + ``` +- Only declare a capability if the plug-in genuinely implements it. For example, if `SupportsClientProvisioning` is `true`, the plug-in must supply a working `IClientProvisioningStore`. + +**Operational reminder:** the Authority host surfaces capability summaries during startup (see `AuthorityIdentityProviderRegistry` log lines). Use those logs during smoke tests to ensure manifests align with expectations. + +**Configuration path normalisation:** Manifest-relative paths (e.g., `tokenSigning.keyDirectory: "../keys"`) are resolved against the YAML file location and environment variables are expanded before validation. Plug-ins should expect to receive an absolute, canonical path when options are injected. + +## 4. Project Scaffold +- Target **.NET 10 preview**, enable nullable, treat warnings as errors, and mark Authority plug-ins with `true`. +- Minimum references: + - `StellaOps.Authority.Plugins.Abstractions` (contracts & capability helpers) + - `StellaOps.Plugin` (hosting/DI helpers) + - `StellaOps.Auth.*` libraries as needed for shared token utilities (optional today). +- Example `.csproj` (trimmed from `StellaOps.Authority.Plugin.Standard`): + ```xml + + + net10.0 + enable + true + true + + + + + + + ``` + (Add other references—e.g., MongoDB driver, shared auth libraries—according to your implementation.) + +## 5. Implementing `IAuthorityPluginRegistrar` +- Create a parameterless registrar class that returns your plug-in type name via `PluginType`. +- Use `AuthorityPluginRegistrationContext` to: + - Bind options (`AddOptions(pluginName).Bind(...)`). + - Register singletons for stores/enrichers using manifest metadata. + - Register any hosted bootstrap tasks (e.g., seed admin users). +- Always validate configuration inside `PostConfigure` and throw meaningful `InvalidOperationException` to fail fast during startup. +- Use the provided `ILoggerFactory` from DI; avoid static loggers or console writes. +- Example skeleton: + ```csharp + internal sealed class MyPluginRegistrar : IAuthorityPluginRegistrar + { + public string PluginType => "my-custom"; + + public void Register(AuthorityPluginRegistrationContext context) + { + var name = context.Plugin.Manifest.Name; + + context.Services.AddOptions(name) + .Bind(context.Plugin.Configuration) + .PostConfigure(opts => opts.Validate(name)); + + context.Services.AddSingleton(sp => + new MyIdentityProvider(context.Plugin, sp.GetRequiredService(), + sp.GetRequiredService(), + sp.GetRequiredService>())); + } + } + ``` + +## 6. Identity Provider Surface +- Implement `IIdentityProviderPlugin` to expose: + - `IUserCredentialStore` for password validation and user CRUD. + - `IClaimsEnricher` to append roles/attributes onto issued principals. + - Optional `IClientProvisioningStore` for machine-to-machine clients. + - `AuthorityIdentityProviderCapabilities` to advertise supported flows. +- Password guidance: + - Prefer Argon2 (Security Guild upcoming recommendation); Standard plug-in currently ships PBKDF2 with easy swap via `IPasswordHasher`. + - Enforce password policies before hashing to avoid storing weak credentials. +- Health checks should probe backing stores (e.g., Mongo `ping`) and return `AuthorityPluginHealthResult` so `/ready` can surface issues. +- When supporting additional factors (e.g., TOTP), implement `SupportsMfa` and document the enrolment flow for resource servers. + +## 7. Configuration & Secrets +- Authority looks for manifests under `etc/authority.plugins/`. Each YAML file maps directly to a plug-in name. +- Support environment overrides using `STELLAOPS_AUTHORITY_PLUGINS__DESCRIPTORS____...`. +- Never store raw secrets in git: allow operators to supply them via `.local.yaml`, environment variables, or injected secret files. Document which keys are mandatory. +- Validate configuration as soon as the registrar runs; use explicit error messages to guide operators. The Standard plug-in now enforces complete bootstrap credentials (username + password) and positive lockout windows via `StandardPluginOptions.Validate`. +- Cross-reference bootstrap workflows with `docs/ops/authority_bootstrap.md` (to be published alongside CORE6) so operators can reuse the same payload formats for manual provisioning. + +## 8. Logging, Metrics, and Diagnostics +- Always log via the injected `ILogger`; include `pluginName` and correlation IDs where available. +- Activity/metric names should align with `AuthorityTelemetry` constants (`service.name=stellaops-authority`). +- Expose additional diagnostics via structured logging rather than writing custom HTTP endpoints; the host will integrate these into `/health` and `/ready`. +- Emit metrics with stable names (`auth.plugins..*`) when introducing custom instrumentation; coordinate with the Observability guild to reserve prefixes. + +## 9. Testing & Tooling +- Unit tests: use Mongo2Go (or similar) to exercise credential stores without hitting production infrastructure (`StandardUserCredentialStoreTests` is a template). +- Determinism: fix timestamps to UTC and sort outputs consistently; avoid random GUIDs unless stable. +- Smoke tests: launch `dotnet run --project src/StellaOps.Authority/StellaOps.Authority` with your plug-in under `PluginBinaries/Authority` and verify `/ready`. +- Example verification snippet: + ```csharp + [Fact] + public async Task VerifyPasswordAsync_ReturnsSuccess() + { + var store = CreateCredentialStore(); + await store.UpsertUserAsync(new AuthorityUserRegistration("alice", "Pa55!", null, null, false, + Array.Empty(), new Dictionary()), CancellationToken.None); + + var result = await store.VerifyPasswordAsync("alice", "Pa55!", CancellationToken.None); + Assert.True(result.Succeeded); + Assert.True(result.User?.Roles.Count == 0); + } + ``` + +## 10. Packaging & Delivery +- Output assembly should follow `StellaOps.Authority.Plugin..dll` so the host’s search pattern picks it up. +- Place the compiled DLL plus dependencies under `PluginBinaries/Authority` for offline deployments; include hashes/signatures in release notes (Security Guild guidance forthcoming). +- Document any external prerequisites (e.g., CA cert bundle) in your plug-in README. +- Update `etc/authority.plugins/.yaml` samples and include deterministic SHA256 hashes for optional bootstrap payloads when distributing Offline Kit artefacts. + +## 11. Checklist & Handoff +- ✅ Capabilities declared and validated in automated tests. +- ✅ Bootstrap workflows documented (if `bootstrap` capability used) and repeatable. +- ✅ Local smoke test + unit/integration suites green (`dotnet test`). +- ✅ Operational docs updated: configuration keys, secrets guidance, troubleshooting. +- Submit the developer guide update referencing PLG6/DOC4 and tag DevEx + Docs reviewers for sign-off. + +--- +**Next documentation actions:** +- Add rendered architectural diagram (PlantUML/mermaid) reflecting the lifecycle above once the Docs toolkit pipeline is ready. +- Reference the LDAP RFC (`docs/rfcs/authority-plugin-ldap.md`) in the capability section once review completes. +- Sync terminology with `docs/11_AUTHORITY.md` when that chapter is published to keep glossary terms consistent. diff --git a/docs/dev/32_AUTH_CLIENT_GUIDE.md b/docs/dev/32_AUTH_CLIENT_GUIDE.md new file mode 100644 index 00000000..48587a41 --- /dev/null +++ b/docs/dev/32_AUTH_CLIENT_GUIDE.md @@ -0,0 +1,91 @@ +# StellaOps Auth Client — Integration Guide + +> **Status:** Drafted 2025-10-10 as part of LIB5. Consumer teams (Feedser, CLI, Agent) should review before wiring the new options into their configuration surfaces. + +The `StellaOps.Auth.Client` library provides a resilient OpenID Connect client for services and tools that talk to **StellaOps Authority**. LIB5 introduced configurable HTTP retry/backoff policies and an offline-fallback window so downstream components stay deterministic even when Authority is briefly unavailable. + +This guide explains how to consume the new settings, when to toggle them, and how to test your integration. + +## 1. Registering the client + +```csharp +services.AddStellaOpsAuthClient(options => +{ + options.Authority = configuration["StellaOps:Authority:Url"]!; + options.ClientId = configuration["StellaOps:Authority:ClientId"]!; + options.ClientSecret = configuration["StellaOps:Authority:ClientSecret"]; + options.DefaultScopes.Add("feedser.jobs.trigger"); + + options.EnableRetries = true; + options.RetryDelays.Clear(); + options.RetryDelays.Add(TimeSpan.FromMilliseconds(500)); + options.RetryDelays.Add(TimeSpan.FromSeconds(2)); + + options.AllowOfflineCacheFallback = true; + options.OfflineCacheTolerance = TimeSpan.FromMinutes(5); +}); +``` + +> **Reminder:** `AddStellaOpsAuthClient` binds the options via `IOptionsMonitor` so changes picked up from configuration reloads will be applied to future HTTP calls without restarting the host. + +## 2. Resilience options + +| Option | Default | Notes | +|--------|---------|-------| +| `EnableRetries` | `true` | When disabled, the shared Polly policy is a no-op and HTTP calls will fail fast. | +| `RetryDelays` | `1s, 2s, 5s` | Edit in ascending order; zero/negative entries are ignored. Clearing the list and leaving it empty keeps the defaults. | +| `AllowOfflineCacheFallback` | `true` | When `true`, stale discovery/JWKS responses are reused within the tolerance window if Authority is unreachable. | +| `OfflineCacheTolerance` | `00:10:00` | Added to the normal cache lifetime. E.g. a 10 minute JWKS cache plus 5 minute tolerance keeps keys for 15 minutes if Authority is offline. | + +The HTTP retry policy handles: + +- 5xx responses +- 429 responses +- Transient transport failures (`HttpRequestException`, timeouts, aborted sockets) + +Retries emit warnings via the `StellaOps.Auth.Client.HttpRetry` logger. Tune the delay values to honour your deployment’s SLOs. + +## 3. Configuration mapping + +Suggested configuration keys (coordinate with consuming teams before finalising): + +```yaml +StellaOps: + Authority: + Url: "https://authority.stella-ops.local" + ClientId: "feedser" + ClientSecret: "change-me" + AuthClient: + EnableRetries: true + RetryDelays: + - "00:00:01" + - "00:00:02" + - "00:00:05" + AllowOfflineCacheFallback: true + OfflineCacheTolerance: "00:10:00" +``` + +Environment variable binding follows the usual double-underscore rules, e.g. + +``` +STELLAOPS__AUTHORITY__AUTHCLIENT__RETRYDELAYS__0=00:00:02 +STELLAOPS__AUTHORITY__AUTHCLIENT__OFFLINECACHETOLERANCE=00:05:00 +``` + +CLI and Feedser teams should expose these knobs once they adopt the auth client. + +## 4. Testing recommendations + +1. **Unit tests:** assert option binding by configuring `StellaOpsAuthClientOptions` via a `ConfigurationBuilder` and ensuring `Validate()` normalises the retry delays and scope list. +2. **Offline fallback:** simulate an unreachable Authority by swapping `HttpMessageHandler` to throw `HttpRequestException` after priming the discovery/JWKS caches. Verify that tokens are still issued until the tolerance expires. +3. **Observability:** watch for `StellaOps.Auth.Client.HttpRetry` warnings in your logs. Excessive retries mean the upstream Authority cluster needs attention. +4. **Determinism:** keep retry delays deterministic. Avoid random jitter—operators can introduce jitter at the infrastructure layer if desired. + +## 5. Rollout checklist + +- [ ] Update consuming service/CLI configuration schema to include the new settings. +- [ ] Document recommended defaults for offline (air-gapped) versus connected deployments. +- [ ] Extend smoke tests to cover Authority outage scenarios. +- [ ] Coordinate with Docs Guild so user-facing quickstarts reference the new knobs. + +Once Feedser and CLI integrate these changes, we can mark LIB5 **DONE**; further packaging work is deferred until the backlog reintroduces it. diff --git a/docs/dev/authority-rate-limit-tuning-outline.md b/docs/dev/authority-rate-limit-tuning-outline.md new file mode 100644 index 00000000..ce8a68ad --- /dev/null +++ b/docs/dev/authority-rate-limit-tuning-outline.md @@ -0,0 +1,33 @@ +# Authority Rate Limit Tuning Outline (2025-10-11) + +## Purpose +- Drive the remaining work on SEC3.B (Security Guild) and PLG6.DOC (Docs Guild) by capturing the agreed baseline for Authority rate limits and related documentation deliverables. +- Provide a single reference for lockout + rate limit interplay so Docs can lift accurate copy into `docs/security/rate-limits.md` and `docs/dev/31_AUTHORITY_PLUGIN_DEVELOPER_GUIDE.md`. + +## Baseline Configuration +- `/token`: fixed window, permitLimit 30, window 60s, queueLimit 0. Reduce to 10/60s for untrusted IP ranges; raise to 60/60s only with compensating controls (WAF + active monitoring). +- `/authorize`: permitLimit 60, window 60s, queueLimit 10. Intended for interactive browser flows; lowering below 30 requires UX review. +- `/internal/*`: disabled by default; recommended 5/60s with queueLimit 0 when bootstrap API exposed. +- Configuration path: `authority.security.rateLimiting.` (e.g., `token.permitLimit`). YAML/ENV bindings follow the standard options hierarchy. +- Retry metadata: middleware stamps `Retry-After` along with tags `authority.client_id`, `authority.remote_ip`, `authority.endpoint`. Docs should highlight these for operator dashboards. + +## Parameter Matrix +| Scenario | permitLimit | window | queueLimit | Notes | +|----------|-------------|--------|------------|-------| +| Default production | 30 | 60s | 0 | Works with anonymous quota (33 scans/day). | +| High-trust clustered IPs | 60 | 60s | 5 | Requires `authorize_rate_limit_hits` alert ≤ 1% sustained. | +| Air-gapped lab | 10 | 120s | 0 | Emphasise reduced concurrency + manual queue draining. | +| Incident lockdown | 5 | 300s | 0 | Pair with lockout lowering to 3 attempts. | + +## Lockout Interplay +- Ensure Docs explain difference between rate limit (per IP/client) vs lockout (per subject). Provide table mapping retry-after headers to recommended support scripts. +- Security Guild to define alert thresholds: trigger SOC ticket when 429 rate > 25% for 5 minutes or when limiter emits >100 events/hour per client. + +## Observability +- Surface metrics: `aspnetcore_rate_limiting_rejections_total{limiter="authority-token"}` and custom log tags from `AuthorityRateLimiterMetadataMiddleware`. +- Recommend dashboard sections: request volume vs. rejections, top offending clientIds, per-endpoint heatmap. + +## Action Items +1. Security Guild (SEC3.B): incorporate matrix + alert rules into `docs/security/rate-limits.md`, add YAML examples for override blocks, and cross-link lockout policy doc. +2. Docs Guild (PLG6.DOC): update developer guide section 9 with the middleware sequence and reference this outline for retry metadata + tuning guidance. +3. Authority Core: validate appsettings sample includes the `security.rateLimiting` block with comments and link back to published doc once ready. diff --git a/docs/dev/merge_semver_playbook.md b/docs/dev/merge_semver_playbook.md new file mode 100644 index 00000000..56774b7d --- /dev/null +++ b/docs/dev/merge_semver_playbook.md @@ -0,0 +1,99 @@ +# Feedser SemVer Merge Playbook (Sprint 1–2) + +This playbook describes how the merge layer and connector teams should emit the new SemVer primitives introduced in Sprint 1–2, how those primitives become normalized version rules, and how downstream jobs query them deterministically. + +## 1. What landed in Sprint 1–2 + +- `RangePrimitives.SemVer` now infers a canonical `style` (`range`, `exact`, `lt`, `lte`, `gt`, `gte`) and captures `exactValue` when the constraint is a single version. +- `NormalizedVersionRule` documents the analytics-friendly projection of each `AffectedPackage` coverage entry and is persisted alongside legacy `versionRanges`. +- `AdvisoryProvenance.decisionReason` records whether merge resolution favored precedence, freshness, or a tie-breaker comparison. + +See `src/StellaOps.Feedser.Models/CANONICAL_RECORDS.md` for the full schema and field descriptions. + +## 2. Mapper pattern + +Connectors should emit SemVer primitives as soon as they can normalize a vendor constraint. The helper `SemVerPrimitiveExtensions.ToNormalizedVersionRule` turns those primitives into the persisted rules: + +```csharp +var primitive = new SemVerPrimitive( + introduced: "1.2.3", + introducedInclusive: true, + fixed: "2.0.0", + fixedInclusive: false, + lastAffected: null, + lastAffectedInclusive: false, + constraintExpression: ">=1.2.3 <2.0.0", + exactValue: null); + +var rule = primitive.ToNormalizedVersionRule(notes: "nvd:CVE-2025-1234"); +// rule => scheme=semver, type=range, min=1.2.3, minInclusive=true, max=2.0.0, maxInclusive=false +``` + +Emit the resulting rule inside `AffectedPackage.NormalizedVersions` while continuing to populate `AffectedVersionRange.RangeExpression` for backward compatibility. + +## 3. Merge dedupe flow + +During merge, feed all package candidates through `NormalizedVersionRuleComparer.Instance` prior to persistence. The comparer orders by scheme → type → min → minInclusive → max → maxInclusive → value → notes, guaranteeing consistent document layout and making `$unwind` pipelines deterministic. + +If multiple connectors emit identical constraints, the merge layer should: + +1. Combine provenance entries (preserving one per source). +2. Preserve a single normalized rule instance (thanks to `NormalizedVersionRuleEqualityComparer.Instance`). +3. Attach `decisionReason="precedence"` if one source overrides another. + +## 4. Example Mongo pipeline + +Use the following aggregation to locate advisories that affect a specific SemVer: + +```javascript +db.advisories.aggregate([ + { $match: { "affectedPackages.type": "semver", "affectedPackages.identifier": "pkg:npm/lodash" } }, + { $unwind: "$affectedPackages" }, + { $unwind: "$affectedPackages.normalizedVersions" }, + { $match: { + $or: [ + { "affectedPackages.normalizedVersions.type": "exact", + "affectedPackages.normalizedVersions.value": "4.17.21" }, + { "affectedPackages.normalizedVersions.type": "range", + "affectedPackages.normalizedVersions.min": { $lte: "4.17.21" }, + "affectedPackages.normalizedVersions.max": { $gt: "4.17.21" } }, + { "affectedPackages.normalizedVersions.type": "gte", + "affectedPackages.normalizedVersions.min": { $lte: "4.17.21" } }, + { "affectedPackages.normalizedVersions.type": "lte", + "affectedPackages.normalizedVersions.max": { $gte: "4.17.21" } } + ] + }}, + { $project: { advisoryKey: 1, title: 1, "affectedPackages.identifier": 1 } } +]); +``` + +Pair this query with the indexes listed in [Normalized Versions Query Guide](mongo_indices.md). + +## 5. Recommended indexes + +| Collection | Index | Purpose | +|------------|-------|---------| +| `advisory` | `{ "affectedPackages.identifier": 1, "affectedPackages.normalizedVersions.scheme": 1, "affectedPackages.normalizedVersions.type": 1 }` (compound, multikey) | Speeds up `$match` on identifier + rule style. | +| `advisory` | `{ "affectedPackages.normalizedVersions.value": 1 }` (sparse) | Optimizes lookups for exact version hits. | + +Coordinate with the Storage team when enabling these indexes so deployment windows account for collection size. + +## 6. Dual-write rollout + +Follow the operational checklist in `docs/ops/migrations/SEMVER_STYLE.md`. The summary: + +1. **Dual write (now)** – emit both legacy `versionRanges` and the new `normalizedVersions`. +2. **Backfill** – follow the storage migration in `docs/ops/migrations/SEMVER_STYLE.md` to rewrite historical advisories before switching consumers. +3. **Verify** – run the aggregation above (with `explain("executionStats")`) to ensure the new indexes are used. +4. **Cutover** – after consumers switch to normalized rules, mark the old `rangeExpression` as deprecated. + +## 7. Checklist for connectors & merge + +- [ ] Populate `SemVerPrimitive` for every SemVer-friendly constraint. +- [ ] Call `ToNormalizedVersionRule` and store the result. +- [ ] Emit provenance masks covering both `versionRanges[].primitives.semver` and `normalizedVersions[]`. +- [ ] Ensure merge deduping relies on the canonical comparer. +- [ ] Capture merge decisions via `decisionReason`. +- [ ] Confirm integration tests include fixtures with normalized rules and SemVer styles. + +For deeper query examples and maintenance tasks, continue with [Normalized Versions Query Guide](mongo_indices.md). diff --git a/docs/dev/mongo_indices.md b/docs/dev/mongo_indices.md new file mode 100644 index 00000000..16a2d9ad --- /dev/null +++ b/docs/dev/mongo_indices.md @@ -0,0 +1,106 @@ +# Normalized Versions Query Guide + +This guide complements the Sprint 1–2 normalized versions rollout. It documents recommended indexes and aggregation patterns for querying `AffectedPackage.normalizedVersions`. + +## 1. Recommended indexes + +When `feedser.storage.enableSemVerStyle` is enabled, advisories expose a flattened +`normalizedVersions` array at the document root. Create these indexes in `mongosh` +after the migration completes (adjust collection name if you use a prefix): + +```javascript +db.advisories.createIndex( + { + "normalizedVersions.packageId": 1, + "normalizedVersions.scheme": 1, + "normalizedVersions.type": 1 + }, + { name: "advisory_normalizedVersions_pkg_scheme_type" } +); + +db.advisories.createIndex( + { "normalizedVersions.value": 1 }, + { name: "advisory_normalizedVersions_value", sparse: true } +); +``` + +- The compound index accelerates `$match` stages that filter by package identifier and rule style without unwinding `affectedPackages`. +- The sparse index keeps storage costs low while supporting pure exact-version lookups (type `exact`). + +The storage bootstrapper creates the same indexes automatically when the feature flag is enabled. + +## 2. Query patterns + +### 2.1 Determine if a specific version is affected + +```javascript +db.advisories.aggregate([ + { $match: { "normalizedVersions.packageId": "pkg:npm/lodash" } }, + { $unwind: "$normalizedVersions" }, + { $match: { + $or: [ + { "normalizedVersions.type": "exact", + "normalizedVersions.value": "4.17.21" }, + { "normalizedVersions.type": "range", + "normalizedVersions.min": { $lte: "4.17.21" }, + "normalizedVersions.max": { $gt: "4.17.21" } }, + { "normalizedVersions.type": "gte", + "normalizedVersions.min": { $lte: "4.17.21" } }, + { "normalizedVersions.type": "lte", + "normalizedVersions.max": { $gte: "4.17.21" } } + ] + }}, + { $project: { advisoryKey: 1, title: 1, "normalizedVersions.packageId": 1 } } +]); +``` + +Use this pipeline during Sprint 2 staging validation runs. Invoke `explain("executionStats")` to confirm the compound index is selected. + +### 2.2 Locate advisories missing normalized rules + +```javascript +db.advisories.aggregate([ + { $match: { $or: [ + { "normalizedVersions": { $exists: false } }, + { "normalizedVersions": { $size: 0 } } + ] } }, + { $project: { advisoryKey: 1, affectedPackages: 1 } } +]); +``` + +Run this query after backfill jobs to identify gaps that still rely solely on `rangeExpression`. + +### 2.3 Deduplicate overlapping rules + +```javascript +db.advisories.aggregate([ + { $unwind: "$normalizedVersions" }, + { $group: { + _id: { + identifier: "$normalizedVersions.packageId", + scheme: "$normalizedVersions.scheme", + type: "$normalizedVersions.type", + min: "$normalizedVersions.min", + minInclusive: "$normalizedVersions.minInclusive", + max: "$normalizedVersions.max", + maxInclusive: "$normalizedVersions.maxInclusive", + value: "$normalizedVersions.value" + }, + advisories: { $addToSet: "$advisoryKey" }, + notes: { $addToSet: "$normalizedVersions.notes" } + }}, + { $match: { "advisories.1": { $exists: true } } }, + { $sort: { "_id.identifier": 1, "_id.type": 1 } } +]); +``` + +Use this to confirm the merge dedupe logic keeps only one normalized rule per unique constraint. + +## 3. Operational checklist + +- [ ] Create the indexes in staging before toggling dual-write in production. +- [ ] Capture explain plans and attach them to the release notes. +- [ ] Notify downstream services that consume advisory snapshots about the new `normalizedVersions` array. +- [ ] Update export fixtures once dedupe verification passes. + +Additional background and mapper examples live in [Feedser SemVer Merge Playbook](merge_semver_playbook.md). diff --git a/docs/license-jwt-quota.md b/docs/license-jwt-quota.md new file mode 100755 index 00000000..f54ee250 --- /dev/null +++ b/docs/license-jwt-quota.md @@ -0,0 +1,123 @@ +--- +title: Offline JWT licence & daily‑run quota +description: How Stella‑Ops enforces a **runs‑per‑day** limit in fully air‑gapped deployments. +nav: + order: 36 +--- + +# JWT‑based daily‑run licence (offline‑capable) + +When *Stella‑Ops* scanners operate entirely **offline**, they cannot phone home +for metering. +Instead, the backend accepts a **signed JSON Web Token (JWT)** that states the +**maximum number of scans per UTC day**. +If no token is supplied, a _grace quota_ of **33 runs/24 h** applies. + +--- + +## 1  Token contents + +| Claim | Purpose | Example | +|-------|---------|---------| +| `sub` | Customer / licensee identifier | `"f47ac10b…"` | +| `iat` | Issued‑at timestamp | `1722566400` | +| `exp` | Absolute licence expiry | `2025‑12‑31T23:59:59Z` | +| `tier` | **Max scans per UTC day** | `{{ quota_token }}` | +| `tid` | Token identifier (32‑byte) | `"7d2285..."` | +| `pkg` | Product SKU / edition | `"stella‑core"` | + +Tokens are signed with **RS256** and verified locally using the bundled public key. +Only the public key ships inside the container; the private key never leaves +the build pipeline. + +--- + +## 2  Obtaining a token + +1. **Request** → `POST /​register { email:"alice@example.org" }` +2. Service hashes the e‑mail (SHA‑256), stores it, and issues a JWT (60 days by default). +3. Token is e‑mailed to you. + +A new request for the same e‑mail returns the **same** token until it nears +expiry, avoiding quota “top‑ups” by re‑registration. + +--- + +## 3  Supplying the token to an air‑gapped stack + +```bash +# recommended +docker run \ + -v /opt/stella/license/alice.jwt:/run/secrets/stella_license.jwt:ro \ + stella‑ops +```` + +Other supported paths: + +| Method | Mount point | Hot‑reload | +| ------------- | ------------------------ | ----------- | +| Docker secret | `/run/secrets/…` | ✓ (inotify) | +| Bind‑mounted | user‑chosen path (above) | ✓ | +| Env variable | `STELLA_LICENSE_JWT` | ✗ restart | + +--- + +## 4  Quota‑enforcement algorithm + +```mermaid +flowchart TD + Start --> Verify[Verify JWT signature] + Verify -->|Invalid| Deny1[Run in non licensed mode] + Verify --> Load[load today's counter UTC] + Load -->|SUM of last 24h scans < daily_quota| Permit[allow scan, add scan] + Permit --> End + Load -->|SUM of last 24h scans ≥ daily_quota| Deny1 +``` + + +## 5  Renewal procedure + +| Scenario | Action | +| -------------- | --------------------------------------------------------------------------------- | +| More capacity | Request new token with higher `daily_quota`; replace file – **no restart needed** | +| Licence expiry | Same as above; new `exp` date | +| Key rotation | Container image ships new public key(s); older tokens still verify | + +--- + +## 6  Fallback limits + +| Situation | Daily quota | +| ----------------------- | ----------------------------------- | +| Valid JWT present | value of `daily_quota` claim ({{ quota_token }}) | +| No JWT | **33** | +| JWT expired (if used) | treated as **anonymous** unless policy enforces hard‑fail | +| Token signature invalid | **0** (reject) | + +--- + +## 7  Threat‑model highlights (future work / optional hardening) + +| Threat | Mitigation | +| --------------------------- | ---------------------------------------------------------------------- | +| Copy token & DB to 2nd node | Bind `sub`/`tid` to host fingerprint (TPM EK) – optional enterprise control | +| Counter DB rollback | Hash‑chain + monotonic clock – optional enterprise control | +| Flooding single node | Redis‑backed cluster rate‑limit (30 hits / 60 s) + edge Nginx (20 r/s) | +| Key compromise | Rotate RS256 key‑pair, ship new pubkey, re‑sign tokens | + +--- + +## 8  Anonymous (33 runs) mode + +Offline PoCs without registration still work: + +```bash +docker compose exec stella-ops stella-jwt reload # reloads, discovers no token +``` + +…but **production deployments *must* register** to unlock real‑world quotas and +receive security advisories via e‑mail. + +--- + +*Last updated: 2025‑08‑02* \ No newline at end of file diff --git a/docs/ops/authority-backup-restore.md b/docs/ops/authority-backup-restore.md new file mode 100644 index 00000000..f587c467 --- /dev/null +++ b/docs/ops/authority-backup-restore.md @@ -0,0 +1,97 @@ +# Authority Backup & Restore Runbook + +## Scope +- **Applies to:** StellaOps Authority deployments running the official `ops/authority/docker-compose.authority.yaml` stack or equivalent Kubernetes packaging. +- **Artifacts covered:** MongoDB (`stellaops-authority` database), Authority configuration (`etc/authority.yaml`), plugin manifests under `etc/authority.plugins/`, and signing key material stored in the `authority-keys` volume (defaults to `/app/keys` inside the container). +- **Frequency:** Run the full procedure prior to upgrades, before rotating keys, and at least once per 24 h in production. Store snapshots in an encrypted, access-controlled vault. + +## Inventory Checklist +| Component | Location (compose default) | Notes | +| --- | --- | --- | +| Mongo data | `mongo-data` volume (`/var/lib/docker/volumes/.../mongo-data`) | Contains all Authority collections (`AuthorityUser`, `AuthorityClient`, `AuthorityToken`, etc.). | +| Configuration | `etc/authority.yaml` | Mounted read-only into the container at `/etc/authority.yaml`. | +| Plugin manifests | `etc/authority.plugins/*.yaml` | Includes `standard.yaml` with `tokenSigning.keyDirectory`. | +| Signing keys | `authority-keys` volume -> `/app/keys` | Path is derived from `tokenSigning.keyDirectory` (defaults to `../keys` relative to the manifest). | + +> **TIP:** Confirm the deployed key directory via `tokenSigning.keyDirectory` in `etc/authority.plugins/standard.yaml`; some installations relocate keys to `/var/lib/stellaops/authority/keys`. + +## Hot Backup (no downtime) +1. **Create output directory:** `mkdir -p backup/$(date +%Y-%m-%d)` on the host. +2. **Dump Mongo:** + ```bash + docker compose -f ops/authority/docker-compose.authority.yaml exec mongo \ + mongodump --archive=/dump/authority-$(date +%Y%m%dT%H%M%SZ).gz \ + --gzip --db stellaops-authority + docker compose -f ops/authority/docker-compose.authority.yaml cp \ + mongo:/dump/authority-$(date +%Y%m%dT%H%M%SZ).gz backup/ + ``` + The `mongodump` archive preserves indexes and can be restored with `mongorestore --archive --gzip`. +3. **Capture configuration + manifests:** + ```bash + cp etc/authority.yaml backup/ + rsync -a etc/authority.plugins/ backup/authority.plugins/ + ``` +4. **Export signing keys:** the compose file maps `authority-keys` to a local Docker volume. Snapshot it without stopping the service: + ```bash + docker run --rm \ + -v authority-keys:/keys \ + -v "$(pwd)/backup:/backup" \ + busybox tar czf /backup/authority-keys-$(date +%Y%m%dT%H%M%SZ).tar.gz -C /keys . + ``` +5. **Checksum:** generate SHA-256 digests for every file and store them alongside the artefacts. +6. **Encrypt & upload:** wrap the backup folder using your secrets management standard (e.g., age, GPG) and upload to the designated offline vault. + +## Cold Backup (planned downtime) +1. Notify stakeholders and drain traffic (CLI clients should refresh tokens afterwards). +2. Stop services: + ```bash + docker compose -f ops/authority/docker-compose.authority.yaml down + ``` +3. Back up volumes directly using `tar`: + ```bash + docker run --rm -v mongo-data:/data -v "$(pwd)/backup:/backup" \ + busybox tar czf /backup/mongo-data-$(date +%Y%m%d).tar.gz -C /data . + docker run --rm -v authority-keys:/keys -v "$(pwd)/backup:/backup" \ + busybox tar czf /backup/authority-keys-$(date +%Y%m%d).tar.gz -C /keys . + ``` +4. Copy configuration + manifests as in the hot backup (steps 3–6). +5. Restart services and verify health: + ```bash + docker compose -f ops/authority/docker-compose.authority.yaml up -d + curl -fsS http://localhost:8080/ready + ``` + +## Restore Procedure +1. **Provision clean volumes:** remove existing volumes if you’re rebuilding a node (`docker volume rm mongo-data authority-keys`), then recreate the compose stack so empty volumes exist. +2. **Restore Mongo:** + ```bash + docker compose exec -T mongo mongorestore --archive --gzip --drop < backup/authority-YYYYMMDDTHHMMSSZ.gz + ``` + Use `--drop` to replace collections; omit if doing a partial restore. +3. **Restore configuration/manifests:** copy `authority.yaml` and `authority.plugins/*` into place before starting the Authority container. +4. **Restore signing keys:** untar into the mounted volume: + ```bash + docker run --rm -v authority-keys:/keys -v "$(pwd)/backup:/backup" \ + busybox tar xzf /backup/authority-keys-YYYYMMDD.tar.gz -C /keys + ``` + Ensure file permissions remain `600` for private keys (`chmod -R 600`). +5. **Start services & validate:** + ```bash + docker compose up -d + curl -fsS http://localhost:8080/health + ``` +6. **Validate JWKS and tokens:** call `/jwks` and issue a short-lived token via the CLI to confirm key material matches expectations. + +## Disaster Recovery Notes +- **Air-gapped replication:** replicate archives via the Offline Update Kit transport channels; never attach USB devices without scanning. +- **Retention:** maintain 30 daily snapshots + 12 monthly archival copies. Rotate encryption keys annually. +- **Key compromise:** if signing keys are suspected compromised, restore from the latest clean backup, rotate via OPS3 (key rotation tooling), and publish a revocation notice. +- **Mongo version:** keep dump/restore images pinned to the deployment version (compose uses `mongo:7`). Restoring across major versions requires a compatibility review. + +## Verification Checklist +- [ ] `/ready` reports all identity providers ready. +- [ ] OAuth flows issue tokens signed by the restored keys. +- [ ] `PluginRegistrationSummary` logs expected providers on startup. +- [ ] Revocation manifest export (`dotnet run --project src/StellaOps.Authority`) succeeds. +- [ ] Monitoring dashboards show metrics resuming (see OPS5 deliverables). + diff --git a/docs/ops/authority-grafana-dashboard.json b/docs/ops/authority-grafana-dashboard.json new file mode 100644 index 00000000..9f4cfc02 --- /dev/null +++ b/docs/ops/authority-grafana-dashboard.json @@ -0,0 +1,174 @@ +{ + "title": "StellaOps Authority - Token & Access Monitoring", + "uid": "authority-token-monitoring", + "schemaVersion": 38, + "version": 1, + "editable": true, + "timezone": "", + "graphTooltip": 0, + "time": { + "from": "now-6h", + "to": "now" + }, + "templating": { + "list": [ + { + "name": "datasource", + "type": "datasource", + "query": "prometheus", + "refresh": 1, + "hide": 0, + "current": {} + } + ] + }, + "panels": [ + { + "id": 1, + "title": "Token Requests – Success vs Failure", + "type": "timeseries", + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "unit": "req/s", + "displayName": "{{grant_type}} ({{status}})" + }, + "overrides": [] + }, + "targets": [ + { + "refId": "A", + "expr": "sum by (grant_type, status) (rate(http_server_duration_seconds_count{service_name=\"stellaops-authority\", http_route=\"/token\"}[5m]))", + "legendFormat": "{{grant_type}} {{status}}" + } + ], + "options": { + "legend": { + "displayMode": "table", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi" + } + } + }, + { + "id": 2, + "title": "Rate Limiter Rejections", + "type": "timeseries", + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "unit": "req/s", + "displayName": "{{limiter}}" + }, + "overrides": [] + }, + "targets": [ + { + "refId": "A", + "expr": "sum by (limiter) (rate(aspnetcore_rate_limiting_rejections_total{service_name=\"stellaops-authority\"}[5m]))", + "legendFormat": "{{limiter}}" + } + ] + }, + { + "id": 3, + "title": "Bypass Events (5m)", + "type": "stat", + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "unit": "short", + "color": { + "mode": "thresholds" + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { "color": "green", "value": null }, + { "color": "orange", "value": 1 }, + { "color": "red", "value": 5 } + ] + } + }, + "overrides": [] + }, + "targets": [ + { + "refId": "A", + "expr": "sum(rate(log_messages_total{message_template=\"Granting StellaOps bypass for remote {RemoteIp}; required scopes {RequiredScopes}.\"}[5m]))" + } + ], + "options": { + "reduceOptions": { + "calcs": ["last"], + "fields": "", + "values": false + }, + "orientation": "horizontal", + "textMode": "auto" + } + }, + { + "id": 4, + "title": "Lockout Events (15m)", + "type": "stat", + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "unit": "short", + "color": { + "mode": "thresholds" + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { "color": "green", "value": null }, + { "color": "orange", "value": 5 }, + { "color": "red", "value": 10 } + ] + } + }, + "overrides": [] + }, + "targets": [ + { + "refId": "A", + "expr": "sum(rate(log_messages_total{message_template=\"Plugin {PluginName} denied access for {Username} due to lockout (retry after {RetryAfter}).\"}[15m]))" + } + ], + "options": { + "reduceOptions": { + "calcs": ["last"], + "fields": "", + "values": false + }, + "orientation": "horizontal", + "textMode": "auto" + } + }, + { + "id": 5, + "title": "Trace Explorer Shortcut", + "type": "text", + "options": { + "mode": "markdown", + "content": "[Open Trace Explorer](#/explore?left={\"datasource\":\"tempo\",\"queries\":[{\"query\":\"{service.name=\\\"stellaops-authority\\\", span_name=~\\\"authority.token.*\\\"}\",\"refId\":\"A\"}]})" + } + } + ], + "links": [] +} diff --git a/docs/ops/authority-monitoring.md b/docs/ops/authority-monitoring.md new file mode 100644 index 00000000..d25c1f35 --- /dev/null +++ b/docs/ops/authority-monitoring.md @@ -0,0 +1,81 @@ +# Authority Monitoring & Alerting Playbook + +## Telemetry Sources +- **Traces:** Activity source `StellaOps.Authority` emits spans for every token flow (`authority.token.validate_*`, `authority.token.handle_*`, `authority.token.validate_access`). Key tags include `authority.endpoint`, `authority.grant_type`, `authority.username`, `authority.client_id`, and `authority.identity_provider`. +- **Metrics:** OpenTelemetry instrumentation (`AddAspNetCoreInstrumentation`, `AddHttpClientInstrumentation`, custom meter `StellaOps.Authority`) exports: + - `http.server.request.duration` histogram (`http_route`, `http_status_code`, `authority.endpoint` tag via `aspnetcore` enrichment). + - `process.runtime.gc.*`, `process.runtime.dotnet.*` (from `AddRuntimeInstrumentation`). +- **Logs:** Serilog writes structured events to stdout. Notable templates: + - `"Password grant verification failed ..."` and `"Plugin {PluginName} denied access ... due to lockout"` (lockout spike detector). + - `"Granting StellaOps bypass for remote {RemoteIp}"` (bypass usage). + - `"Rate limit exceeded for path {Path} from {RemoteIp}"` (limiter alerts). + +## Prometheus Metrics to Collect +| Metric | Query | Purpose | +| --- | --- | --- | +| `token_requests_total` | `sum by (grant_type, status) (rate(http_server_duration_seconds_count{service_name="stellaops-authority", http_route="/token"}[5m]))` | Token issuance volume per grant type (`grant_type` comes via `authority.grant_type` span attribute → Exemplars in Grafana). | +| `token_failure_ratio` | `sum(rate(http_server_duration_seconds_count{service_name="stellaops-authority", http_route="/token", http_status_code=~"4..|5.."}[5m])) / sum(rate(http_server_duration_seconds_count{service_name="stellaops-authority", http_route="/token"}[5m]))` | Alert when > 5 % for 10 min. | +| `authorize_rate_limit_hits` | `sum(rate(aspnetcore_rate_limiting_rejections_total{service_name="stellaops-authority", limiter="authority-token"}[5m]))` | Detect rate limiting saturations (requires OTEL ASP.NET rate limiter exporter). | +| `lockout_events` | `sum by (plugin) (rate(log_messages_total{app="stellaops-authority", level="Warning", message_template="Plugin {PluginName} denied access for {Username} due to lockout (retry after {RetryAfter})."}[5m]))` | Derived from Loki/Promtail log counter. | +| `bypass_usage_total` | `sum(rate(log_messages_total{app="stellaops-authority", level="Information", message_template="Granting StellaOps bypass for remote {RemoteIp}; required scopes {RequiredScopes}."}[5m]))` | Track trusted bypass invocations. | + +> **Exporter note:** Enable `aspnetcore` meters (`dotnet-counters` name `Microsoft.AspNetCore.Hosting`), or configure the OpenTelemetry Collector `metrics` pipeline with `metric_statements` to remap histogram counts into the shown series. + +## Alert Rules +1. **Token Failure Surge** + - _Expression_: `token_failure_ratio > 0.05` + - _For_: `10m` + - _Labels_: `severity="critical"` + - _Annotations_: Include `topk(5, sum by (authority_identity_provider) (increase(authority_token_rejections_total[10m])))` as diagnostic hint (requires span → metric transformation). +2. **Lockout Spike** + - _Expression_: `sum(rate(log_messages_total{message_template="Plugin {PluginName} denied access for {Username} due to lockout (retry after {RetryAfter})."}[15m])) > 10` + - _For_: `15m` + - Investigate credential stuffing; consider temporarily tightening `RateLimiting.Token`. +3. **Bypass Threshold** + - _Expression_: `sum(rate(log_messages_total{message_template="Granting StellaOps bypass for remote {RemoteIp}; required scopes {RequiredScopes}."}[5m])) > 1` + - _For_: `5m` + - Alert severity `warning` — verify the calling host list. +4. **Rate Limiter Saturation** + - _Expression_: `sum(rate(aspnetcore_rate_limiting_rejections_total{service_name="stellaops-authority"}[5m])) > 0` + - Escalate if sustained for 5 min; confirm trusted clients aren’t misconfigured. + +## Grafana Dashboard +- Import `docs/ops/authority-grafana-dashboard.json` to provision baseline panels: + - **Token Success vs Failure** – stacked rate visualization split by grant type. + - **Rate Limiter Hits** – bar chart showing `authority-token` and `authority-authorize`. + - **Bypass & Lockout Events** – dual-stat panel using Loki-derived counters. + - **Trace Explorer Link** – panel links to `StellaOps.Authority` span search pre-filtered by `authority.grant_type`. + +## Collector Configuration Snippets +```yaml +receivers: + otlp: + protocols: + http: +exporters: + prometheus: + endpoint: "0.0.0.0:9464" +processors: + batch: + attributes/token_grant: + actions: + - key: grant_type + action: upsert + from_attribute: authority.grant_type +service: + pipelines: + metrics: + receivers: [otlp] + processors: [attributes/token_grant, batch] + exporters: [prometheus] + logs: + receivers: [otlp] + processors: [batch] + exporters: [loki] +``` + +## Operational Checklist +- [ ] Confirm `STELLAOPS_AUTHORITY__OBSERVABILITY__EXPORTERS` enables OTLP in production builds. +- [ ] Ensure Promtail captures container stdout with Serilog structured formatting. +- [ ] Periodically validate alert noise by running load tests that trigger the rate limiter. +- [ ] Include dashboard JSON in Offline Kit for air-gapped clusters; update version header when metrics change. diff --git a/docs/ops/feedser-conflict-resolution.md b/docs/ops/feedser-conflict-resolution.md new file mode 100644 index 00000000..255f09a7 --- /dev/null +++ b/docs/ops/feedser-conflict-resolution.md @@ -0,0 +1,130 @@ +# Feedser Conflict Resolution Runbook (Sprint 3) + +This runbook equips Feedser operators to detect, triage, and resolve advisory conflicts now that the Sprint 3 merge engine landed (`AdvisoryPrecedenceMerger`, merge-event hashing, and telemetry counters). It builds on the canonical rules defined in `src/DEDUP_CONFLICTS_RESOLUTION_ALGO.md` and the metrics/logging instrumentation delivered this sprint. + +--- + +## 1. Precedence Model (recap) + +- **Default ranking:** `GHSA -> NVD -> OSV`, with distro/vendor PSIRTs outranking ecosystem feeds (`AdvisoryPrecedenceDefaults`). Use `feedser:merge:precedence:ranks` to override per source when incident response requires it. +- **Freshness override:** if a lower-ranked source is >= 48 hours newer for a freshness-sensitive field (title, summary, affected ranges, references, credits), it wins. Every override stamps `provenance[].decisionReason = freshness`. +- **Tie-breakers:** when precedence and freshness tie, the engine falls back to (1) primary source order, (2) shortest normalized text, (3) lowest stable hash. Merge-generated provenance records set `decisionReason = tie-breaker`. +- **Audit trail:** each merged advisory receives a `merge` provenance entry listing the participating sources plus a `merge_event` record with canonical before/after SHA-256 hashes. + +--- + +## 2. Telemetry Shipped This Sprint + +| Instrument | Type | Key Tags | Purpose | +|------------|------|----------|---------| +| `feedser.merge.operations` | Counter | `inputs` | Total precedence merges executed. | +| `feedser.merge.overrides` | Counter | `primary_source`, `suppressed_source`, `primary_rank`, `suppressed_rank` | Field-level overrides chosen by precedence. | +| `feedser.merge.range_overrides` | Counter | `advisory_key`, `package_type`, `primary_source`, `suppressed_source`, `primary_range_count`, `suppressed_range_count` | Package range overrides emitted by `AffectedPackagePrecedenceResolver`. | +| `feedser.merge.conflicts` | Counter | `type` (`severity`, `precedence_tie`), `reason` (`mismatch`, `primary_missing`, `equal_rank`) | Conflicts requiring operator review. | +| `feedser.merge.identity_conflicts` | Counter | `scheme`, `alias_value`, `advisory_count` | Alias collisions surfaced by the identity graph. | + +### Structured logs + +- `AdvisoryOverride` (EventId 1000) - logs merge suppressions with alias/provenance counts. +- `PackageRangeOverride` (EventId 1001) - logs package-level precedence decisions. +- `PrecedenceConflict` (EventId 1002) - logs mismatched severity or equal-rank scenarios. +- `Alias collision ...` (no EventId) - emitted when `feedser.merge.identity_conflicts` increments. + +Expect all logs at `Information`. Ensure OTEL exporters include the scope `StellaOps.Feedser.Merge`. + +--- + +## 3. Detection & Alerting + +1. **Dashboard panels** + - `feedser.merge.conflicts` - table grouped by `type/reason`. Alert when > 0 in a 15 minute window. + - `feedser.merge.range_overrides` - stacked bar by `package_type`. Spikes highlight vendor PSIRT overrides over registry data. + - `feedser.merge.overrides` with `primary_source|suppressed_source` - catches unexpected precedence flips (e.g., OSV overtaking GHSA). + - `feedser.merge.identity_conflicts` - single-stat; alert when alias collisions occur more than once per day. +2. **Log based alerts** + - `eventId=1002` with `reason="equal_rank"` - indicates precedence table gaps; page merge owners. + - `eventId=1002` with `reason="mismatch"` - severity disagreement; open connector bug if sustained. +3. **Job health** + - `stellaops-cli db merge` exit code `1` signifies unresolved conflicts. Pipe to automation that captures logs and notifies #feedser-ops. + +--- + +## 4. Triage Workflow + +1. **Confirm job context** + - `stellaops-cli db merge` (CLI) or `POST /jobs/merge:reconcile` (API) to rehydrate the merge job. Use `--verbose` to stream structured logs during triage. +2. **Inspect metrics** + - Correlate spikes in `feedser.merge.conflicts` with `primary_source`/`suppressed_source` tags from `feedser.merge.overrides`. +3. **Pull structured logs** + - Example (vector output): + ``` + jq 'select(.EventId.Name=="PrecedenceConflict") | {advisory: .State[0].Value, type: .ConflictType, reason: .Reason, primary: .PrimarySources, suppressed: .SuppressedSources}' stellaops-feedser.log + ``` +4. **Review merge events** + - `mongosh`: + ```javascript + use feedser; + db.merge_event.find({ advisoryKey: "CVE-2025-1234" }).sort({ mergedAt: -1 }).limit(5); + ``` + - Compare `beforeHash` vs `afterHash` to confirm the merge actually changed canonical output. +5. **Interrogate provenance** + - `db.advisories.findOne({ advisoryKey: "CVE-2025-1234" }, { title: 1, severity: 1, provenance: 1, "affectedPackages.provenance": 1 })` + - Check `provenance[].decisionReason` values (`precedence`, `freshness`, `tie-breaker`) to understand why the winning field was chosen. + +--- + +## 5. Conflict Classification Matrix + +| Signal | Likely Cause | Immediate Action | +|--------|--------------|------------------| +| `reason="mismatch"` with `type="severity"` | Upstream feeds disagree on CVSS vector/severity. | Verify which feed is freshest; if correctness is known, adjust connector mapping or precedence override. | +| `reason="primary_missing"` | Higher-ranked source lacks the field entirely. | Backfill connector data or temporarily allow lower-ranked source via precedence override. | +| `reason="equal_rank"` | Two feeds share the same precedence rank (custom config or missing entry). | Update `feedser:merge:precedence:ranks` to break the tie; restart merge job. | +| Rising `feedser.merge.range_overrides` for a package type | Vendor PSIRT now supplies richer ranges. | Validate connectors emit `decisionReason="precedence"` and update dashboards to treat registry ranges as fallback. | +| `feedser.merge.identity_conflicts` > 0 | Alias scheme mapping produced collisions (duplicate CVE <-> advisory pairs). | Inspect `Alias collision` log payload; reconcile the alias graph by adjusting connector alias output. | + +--- + +## 6. Resolution Playbook + +1. **Connector data fix** + - Re-run the offending connector stages (`stellaops-cli db fetch --source ghsa --stage map` etc.). + - Once fixed, rerun merge and verify `decisionReason` reflects `freshness` or `precedence` as expected. +2. **Temporary precedence override** + - Edit `etc/feedser.yaml`: + ```yaml + feedser: + merge: + precedence: + ranks: + osv: 1 + ghsa: 0 + ``` + - Restart Feedser workers; confirm tags in `feedser.merge.overrides` show the new ranks. + - Document the override with expiry in the change log. +3. **Alias remediation** + - Update connector mapping rules to weed out duplicate aliases (e.g., skip GHSA aliases that mirror CVE IDs). + - Flush cached alias graphs if necessary (`db.alias_graph.drop()` is destructive-coordinate with Storage before issuing). +4. **Escalation** + - If override metrics spike due to upstream regression, open an incident with Security Guild, referencing merge logs and `merge_event` IDs. + +--- + +## 7. Validation Checklist + +- [ ] Merge job rerun returns exit code `0`. +- [ ] `feedser.merge.conflicts` baseline returns to zero after corrective action. +- [ ] Latest `merge_event` entry shows expected hash delta. +- [ ] Affected advisory document shows updated `provenance[].decisionReason`. +- [ ] Ops change log updated with incident summary, config overrides, and rollback plan. + +--- + +## 8. Reference Material + +- Canonical conflict rules: `src/DEDUP_CONFLICTS_RESOLUTION_ALGO.md`. +- Merge engine internals: `src/StellaOps.Feedser.Merge/Services/AdvisoryPrecedenceMerger.cs`. +- Metrics definitions: `src/StellaOps.Feedser.Merge/Services/AdvisoryMergeService.cs` (identity conflicts) and `AdvisoryPrecedenceMerger`. +- Storage audit trail: `src/StellaOps.Feedser.Merge/Services/MergeEventWriter.cs`, `src/StellaOps.Feedser.Storage.Mongo/MergeEvents`. + +Keep this runbook synchronized with future sprint notes and update alert thresholds as baseline volumes change. diff --git a/docs/ops/feedser-cve-kev-operations.md b/docs/ops/feedser-cve-kev-operations.md new file mode 100644 index 00000000..8635933c --- /dev/null +++ b/docs/ops/feedser-cve-kev-operations.md @@ -0,0 +1,104 @@ +# Feedser CVE & KEV Connector Operations + +This playbook equips operators with the steps required to roll out and monitor the CVE Services and CISA KEV connectors across environments. + +## 1. CVE Services Connector (`source:cve:*`) + +### 1.1 Prerequisites + +- CVE Services API credentials (organisation ID, user ID, API key) with access to the JSON 5 API. +- Network egress to `https://cveawg.mitre.org` (or a mirrored endpoint) from the Feedser workers. +- Updated `feedser.yaml` (or the matching environment variables) with the following section: + +```yaml +feedser: + sources: + cve: + baseEndpoint: "https://cveawg.mitre.org/api/" + apiOrg: "ORG123" + apiUser: "user@example.org" + apiKeyFile: "/var/run/secrets/feedser/cve-api-key" + pageSize: 200 + maxPagesPerFetch: 5 + initialBackfill: "30.00:00:00" + requestDelay: "00:00:00.250" + failureBackoff: "00:10:00" +``` + +> ℹ️ Store the API key outside source control. When using `apiKeyFile`, mount the secret file into the container/host; alternatively supply `apiKey` via `FEEDSER_SOURCES__CVE__APIKEY`. + +### 1.2 Smoke Test (staging) + +1. Deploy the updated configuration and restart the Feedser service so the connector picks up the credentials. +2. Trigger one end-to-end cycle: + - Feedser CLI: `stella db jobs run source:cve:fetch --and-then source:cve:parse --and-then source:cve:map` + - REST fallback: `POST /jobs/run { "kind": "source:cve:fetch", "chain": ["source:cve:parse", "source:cve:map"] }` +3. Observe the following metrics (exported via OTEL meter `StellaOps.Feedser.Source.Cve`): + - `cve.fetch.attempts`, `cve.fetch.success`, `cve.fetch.failures`, `cve.fetch.unchanged` + - `cve.parse.success`, `cve.parse.failures`, `cve.parse.quarantine` + - `cve.map.success` +4. Verify the MongoDB advisory store contains fresh CVE advisories (`advisoryKey` prefix `cve/`) and that the source cursor (`source_states` collection) advanced. + +### 1.3 Production Monitoring + +- **Dashboards** – Add the counters above plus `feedser.range.primitives` (filtered by `scheme=semver` or `scheme=vendor`) to the Feedser overview board. Alert when: + - `rate(cve.fetch.failures[5m]) > 0` + - `rate(cve.map.success[15m]) == 0` while fetch attempts continue + - `sum_over_time(cve.parse.quarantine[1h]) > 0` +- **Logs** – Watch for `CveConnector` warnings such as `Failed fetching CVE record` or schema validation errors (`Malformed CVE JSON`). These are emitted with the CVE ID and document identifier for triage. +- **Backfill window** – operators can tighten or widen the `initialBackfill` / `maxPagesPerFetch` values after validating baseline throughput. Update the config and restart the worker to apply changes. + +## 2. CISA KEV Connector (`source:kev:*`) + +### 2.1 Prerequisites + +- Network egress (or mirrored content) for `https://www.cisa.gov/sites/default/files/feeds/known_exploited_vulnerabilities.json`. +- No credentials are required, but the HTTP allow-list must include `www.cisa.gov`. +- Confirm the following snippet in `feedser.yaml` (defaults shown; tune as needed): + +```yaml +feedser: + sources: + kev: + feedUri: "https://www.cisa.gov/sites/default/files/feeds/known_exploited_vulnerabilities.json" + requestTimeout: "00:01:00" + failureBackoff: "00:05:00" +``` + +### 2.2 Schema validation & anomaly handling + +From this sprint the connector validates the KEV JSON payload against `Schemas/kev-catalog.schema.json`. Malformed documents are quarantined, and entries missing a CVE ID are dropped with a warning (`reason=missingCveId`). Operators should treat repeated schema failures as an upstream regression and coordinate with CISA or mirror maintainers. + +### 2.3 Smoke Test (staging) + +1. Deploy the configuration and restart Feedser. +2. Trigger a pipeline run: + - CLI: `stella db jobs run source:kev:fetch --and-then source:kev:parse --and-then source:kev:map` + - REST: `POST /jobs/run { "kind": "source:kev:fetch", "chain": ["source:kev:parse", "source:kev:map"] }` +3. Verify the metrics exposed by meter `StellaOps.Feedser.Source.Kev`: + - `kev.fetch.attempts`, `kev.fetch.success`, `kev.fetch.unchanged`, `kev.fetch.failures` + - `kev.parse.entries` (tag `catalogVersion`), `kev.parse.failures`, `kev.parse.anomalies` (tag `reason`) + - `kev.map.advisories` (tag `catalogVersion`) +4. Confirm MongoDB documents exist for the catalog JSON (`raw_documents` & `dtos`) and that advisories with prefix `kev/` are written. + +### 2.4 Production Monitoring + +- Alert when `kev.fetch.success` goes to zero for longer than the expected daily cadence (default: trigger if `rate(kev.fetch.success[8h]) == 0` during business hours). +- Track anomaly spikes via `kev.parse.anomalies{reason="missingCveId"}`. A sustained non-zero rate means the upstream catalog contains unexpected records. +- The connector logs each validated catalog: `Parsed KEV catalog document … entries=X`. Absence of that log alongside consecutive `kev.fetch.success` counts suggests schema validation failures—correlate with warning-level events in the `StellaOps.Feedser.Source.Kev` logger. + +### 2.5 Known good dashboard tiles + +Add the following panels to the Feedser observability board: + +| Metric | Recommended visualisation | +|--------|---------------------------| +| `kev.fetch.success` | Single-stat (last 24 h) with threshold alert | +| `rate(kev.parse.entries[1h])` by `catalogVersion` | Stacked area – highlights daily release size | +| `sum_over_time(kev.parse.anomalies[1d])` by `reason` | Table – anomaly breakdown | + +## 3. Runbook updates + +- Record staging/production smoke test results (date, catalog version, advisory counts) in your team’s change log. +- Add the CVE/KEV job kinds to the standard maintenance checklist so operators can manually trigger them after planned downtime. +- Keep this document in sync with future connector changes (for example, new anomaly reasons or additional metrics). diff --git a/docs/ops/migrations/SEMVER_STYLE.md b/docs/ops/migrations/SEMVER_STYLE.md new file mode 100644 index 00000000..151f078b --- /dev/null +++ b/docs/ops/migrations/SEMVER_STYLE.md @@ -0,0 +1,50 @@ +# SemVer Style Backfill Runbook + +_Last updated: 2025-10-11_ + +## Overview + +The SemVer style migration populates the new `normalizedVersions` field on advisory documents and ensures +provenance `decisionReason` values are preserved during future reads. The migration is idempotent and only +runs when the feature flag `feedser:storage:enableSemVerStyle` is enabled. + +## Preconditions + +1. **Review configuration** – set `feedser.storage.enableSemVerStyle` to `true` on all Feedser services. +2. **Confirm batch size** – adjust `feedser.storage.backfillBatchSize` if you need smaller batches for older + deployments (default: `250`). +3. **Back up** – capture a fresh snapshot of the `advisory` collection or a full MongoDB backup. +4. **Staging dry-run** – enable the flag in a staging environment and observe the migration output before + rolling to production. + +## Execution + +No manual command is required. After deploying the configuration change, restart the Feedser WebService or +any component that hosts the Mongo migration runner. During startup you will see log entries similar to: + +``` +Applying Mongo migration 20251011-semver-style-backfill: Populate advisory.normalizedVersions for existing documents when SemVer style storage is enabled. +Mongo migration 20251011-semver-style-backfill applied +``` + +The migration reads advisories in batches (`feedser.storage.backfillBatchSize`) and writes flattened +`normalizedVersions` arrays. Existing documents without SemVer ranges remain untouched. + +## Post-checks + +1. Verify the new indexes exist: + ``` + db.advisory.getIndexes() + ``` + You should see `advisory_normalizedVersions_pkg_scheme_type` and `advisory_normalizedVersions_value`. +2. Spot check a few advisories to confirm the top-level `normalizedVersions` array exists and matches + the embedded package data. +3. Run `dotnet test` for `StellaOps.Feedser.Storage.Mongo.Tests` (optional but recommended) in CI to confirm + the storage suite passes with the feature flag enabled. + +## Rollback + +Set `feedser.storage.enableSemVerStyle` back to `false` and redeploy. The migration will be skipped on +subsequent startups. You can leave the populated `normalizedVersions` arrays in place; they are ignored when +the feature flag is off. If you must remove them entirely, restore from the backup captured during +preparation. diff --git a/docs/rfcs/authority-plugin-ldap.md b/docs/rfcs/authority-plugin-ldap.md new file mode 100644 index 00000000..9e965bfd --- /dev/null +++ b/docs/rfcs/authority-plugin-ldap.md @@ -0,0 +1,136 @@ +# RFC: StellaOps.Authority.Plugin.Ldap + +**Status:** Draft – for review by Auth Guild, Security Guild, DevEx (2025-10-10) +**Authors:** Plugin Team 4 (Auth Libraries & Identity Providers) +**Related initiatives:** PLG7 backlog, CORE5 event handlers, DOC4 developer guide + +## 1. Problem Statement +Many on-prem StellaOps deployments rely on existing LDAP/Active Directory domains for workforce identity. The current Standard Mongo-backed plugin requires duplicating users and secrets, which increases operational overhead and violates corporate policy in some regulated environments. We need a sovereign, offline-friendly LDAP plugin that: + +- Supports password grant and bootstrap provisioning flows without storing credentials in Mongo. +- Enforces StellaOps security policies (lockout, password policy hints, audit logging) while delegating credential validation to LDAP. +- Operates deterministically in offline or partially connected environments by caching directory metadata when necessary. + +## 2. Goals +- Provide a first-party `StellaOps.Authority.Plugin.Ldap` plugin advertising `password` and optional `clientProvisioning` capabilities at launch. +- Support username/password authentication against LDAP bind operations with configurable DN templates. +- Allow optional bootstrap seeding of service accounts by writing into LDAP (guarded behind explicit configuration) or by mapping to pre-existing entries. +- Surface directory-derived claims (groups, attributes) for downstream authorization via `IClaimsEnricher`. +- Integrate with Authority lockout telemetry and structured logging without persisting secrets locally. + +## 3. Non-Goals +- Implement multi-factor authentication out of the box (future enhancement once TOTP/WebAuthn strategy is finalised). +- Provide write-heavy directory management (e.g., user creation workflows) beyond optional bootstrap service account seeding. +- Replace the Standard plugin; both must remain supported and selectable per environment. + +## 4. Key Constraints & Assumptions +- Offline-first posture: deployments may operate without outbound internet and with intermittent directory connectivity (e.g., read-only replicas). The plugin must tolerate transient LDAP connectivity failures and degrade gracefully. +- Deterministic behaviour: identical configuration and directory state must yield identical token issuance results. Cached metadata (e.g., group lookups) must have defined expiration. +- Security: No plaintext credential storage; TLS must be enforced for LDAP connections unless explicitly overridden for air-gapped lab environments. + +## 5. High-Level Architecture +1. **Configuration binding** (`ldap.yaml`): defines server endpoints, bind strategy, claim mapping, and optional bootstrap overrides. +2. **Connection factory**: pooled LDAP connections using a resilient client (preferred dependency: `Novell.Directory.Ldap.NETStandard`). +3. **Credential validator** (`IUserCredentialStore`): performs bind-as-user flow with optional fallback bind using service account when directories disallow anonymous search. +4. **Claims enricher** (`IClaimsEnricher`): queries group membership/attributes and projects them into canonical roles/claims. +5. **Optional client provisioning** (`IClientProvisioningStore`): maintains machine/service principals either in Mongo (metadata) or via LDAP `serviceConnectionPoint` entries based on configuration. +6. **Health checks**: periodic LDAP `whoami` or `search` probes surfaced through `AuthorityPluginHealthResult`. + +``` +Authority Host + ├── Plugin Manifest (ldap) + ├── Registrar → registers ConnectionFactory, LdapCredentialStore, LdapClaimsEnricher + ├── Password Grant Handler → CredentialStore.VerifyPasswordAsync → LDAP Bind + └── Claims Pipeline → ClaimsEnricher.EnrichAsync → LDAP group lookup +``` + +## 6. Configuration Schema (Draft) +```yaml +connection: + host: "ldaps://ldap.example.internal" + port: 636 + useStartTls: false + validateCertificates: true + bindDn: "cn=stellaops-bind,ou=service,dc=example,dc=internal" + bindPasswordSecret: "file:/etc/stellaops/secrets/ldap-bind.txt" + searchBase: "dc=example,dc=internal" + usernameAttribute: "uid" + userDnFormat: "uid={username},ou=people,dc=example,dc=internal" # optional template +security: + requireTls: true + allowedCipherSuites: [] # optional allow-list + referralChasing: false +lockout: + useAuthorityPolicies: true # reuse Authority lockout counters + directoryLockoutAttribute: "pwdAccountLockedTime" +claims: + groupAttribute: "memberOf" + groupToRoleMap: + "cn=stellaops-admins,ou=groups,dc=example,dc=internal": "operators" + "cn=stellaops-read,ou=groups,dc=example,dc=internal": "auditors" + extraAttributes: + displayName: "displayName" + email: "mail" +clientProvisioning: + enabled: false + containerDn: "ou=service,dc=example,dc=internal" + secretAttribute: "userPassword" +health: + probeIntervalSeconds: 60 + timeoutSeconds: 5 +``` + +## 7. Capability Mapping +| Capability | Implementation Notes | +|------------|---------------------| +| `password` | Bind-as-user validation with Authority lockout integration. Mandatory. | +| `clientProvisioning` | Optional; when enabled, creates/updates LDAP entries for machine clients or stores metadata in Mongo if directory writes are disabled. | +| `bootstrap` | Exposed only when bootstrap manifest provides service account credentials AND directory write permissions are confirmed during startup. | +| `mfa` | Not supported in MVP. Future iteration may integrate TOTP attributes or external MFA providers. | + +## 8. Operational Considerations +- **Offline cache:** provide optional Mongo cache for group membership to keep `/ready` responsive if LDAP is temporarily unreachable. Cache entries must include TTL and invalidation hooks. +- **Secrets management:** accept `file:` and environment variable references; integrate with existing `StellaOps.Configuration` secret providers. +- **Observability:** emit structured logs with event IDs (`LDAP_BIND_START`, `LDAP_BIND_FAILURE`, `LDAP_GROUP_LOOKUP`), counters for success/failure, and latency histograms. +- **Throttling:** reuse Authority rate-limiting middleware; add per-connection throttles to avoid saturating directory servers during brute-force attacks. + +## 9. Security & Compliance +- Enforce TLS (`ldaps://` or STARTTLS) by default. Provide explicit `allowInsecure` flag gated behind environment variable for lab/testing only. +- Support password hash migration by detecting directory lockout attributes and surfacing `RequiresPasswordReset` when policies demand changes. +- Log distinguished names only at `Debug` level to avoid leaking sensitive structure in default logs. +- Coordinate with Security Guild for penetration testing before GA; incorporate audit log entries for bind attempts and provisioning changes. + +## 10. Testing Strategy +- **Unit tests:** mock LDAP connections to validate DN formatting, error mapping, and capability negotiation. +- **Integration tests:** run against an ephemeral OpenLDAP container (seeded via LDIF fixtures) within CI. Include offline cache regression (disconnect LDAP mid-test). +- **Determinism tests:** feed identical LDIF snapshots and configuration to ensure output tokens/claims remain stable across runs. +- **Smoke tests:** `dotnet test` harness plus manual `dotnet run` scenario verifying `/token` password grants and `/internal/users` bootstrap with LDAP-backed store. + +## 11. Implementation Plan +1. Scaffold `StellaOps.Authority.Plugin.Ldap` project + tests (net10.0, `` true). +2. Implement configuration options + validation (mirroring Standard plugin guardrails). +3. Build connection factory + credential store with bind logic. +4. Implement claims enricher and optional cache layer. +5. Add client provisioning store (optional) with toggles for read-only deployments. +6. Wire bootstrapper to validate connectivity/permissions and record findings in startup logs. +7. Extend developer guide with LDAP specifics (post-RFC acceptance). +8. Update Docs and TODO trackers; produce release notes entry once merged. + +## 12. Open Questions +- Should client provisioning default to storing metadata in Mongo even when LDAP writes succeed (to preserve audit history)? +- Do we require LDAPS mutual TLS support (client certificates) for regulated environments? If yes, need to extend configuration schema. +- How will we map LDAP groups to Authority scopes/roles when names differ significantly? Consider supporting regex or mapping scripts. + +## 13. Timeline (Tentative) +- **Week 1:** RFC review & sign-off. +- **Week 2-3:** Implementation & unit tests. +- **Week 4:** Integration tests + documentation updates. +- **Week 5:** Security review, release candidate packaging. + +## 14. Approval +- **Auth Guild Lead:** _TBD_ +- **Security Guild Representative:** _TBD_ +- **DevEx Docs:** _TBD_ + +--- +Please add comments inline or via PR review. Once approved, track execution under PLG7. diff --git a/etc/authority.plugins/ldap.yaml b/etc/authority.plugins/ldap.yaml new file mode 100644 index 00000000..30e9a4a1 --- /dev/null +++ b/etc/authority.plugins/ldap.yaml @@ -0,0 +1,17 @@ +# Placeholder configuration for the LDAP identity provider plug-in. +# Replace values with your directory settings before enabling the plug-in. +connection: + host: "ldap.example.com" + port: 636 + useTls: true + bindDn: "cn=service,dc=example,dc=com" + bindPassword: "CHANGE_ME" + +queries: + userFilter: "(uid={username})" + groupFilter: "(member={distinguishedName})" + groupAttribute: "cn" + +capabilities: + supportsPassword: true + supportsMfa: false diff --git a/etc/authority.plugins/standard.yaml b/etc/authority.plugins/standard.yaml new file mode 100644 index 00000000..e2273e86 --- /dev/null +++ b/etc/authority.plugins/standard.yaml @@ -0,0 +1,22 @@ +# Standard plugin configuration (Mongo-backed identity store). +bootstrapUser: + username: "admin" + password: "changeme" + +passwordPolicy: + minimumLength: 12 + requireUppercase: true + requireLowercase: true + requireDigit: true + requireSymbol: true + +lockout: + enabled: true + maxAttempts: 5 + windowMinutes: 15 + +tokenSigning: + # Path to the directory containing signing keys (relative paths resolve + # against the location of this manifest, environment variables are expanded, + # and the final value is normalised to an absolute path during startup. + keyDirectory: "../keys" diff --git a/etc/authority.yaml.sample b/etc/authority.yaml.sample new file mode 100644 index 00000000..957d1cd5 --- /dev/null +++ b/etc/authority.yaml.sample @@ -0,0 +1,71 @@ +# StellaOps Authority configuration template. +# Copy to ../etc/authority.yaml (relative to the Authority content root) +# and adjust values to fit your environment. Environment variables +# prefixed with STELLAOPS_AUTHORITY_ override these values at runtime. +# Example: STELLAOPS_AUTHORITY__ISSUER=https://authority.example.com + +schemaVersion: 1 + +# Absolute issuer URI advertised to clients. Use HTTPS for anything +# beyond loopback development. +issuer: "https://authority.stella-ops.local" + +# Token lifetimes expressed as HH:MM:SS or DD.HH:MM:SS. +accessTokenLifetime: "00:15:00" +refreshTokenLifetime: "30.00:00:00" +identityTokenLifetime: "00:05:00" +authorizationCodeLifetime: "00:05:00" +deviceCodeLifetime: "00:15:00" + +# MongoDB storage connection details. +storage: + connectionString: "mongodb://localhost:27017/stellaops-authority" + # databaseName: "stellaops_authority" + commandTimeout: "00:00:30" + +# Bootstrap administrative endpoints (initial provisioning). +bootstrap: + enabled: false + apiKey: "change-me" + defaultIdentityProvider: "standard" + +# Directories scanned for Authority plug-ins. Relative paths resolve +# against the application content root, enabling air-gapped deployments +# that package plug-ins alongside binaries. +pluginDirectories: + - "../PluginBinaries/Authority" + # "/var/lib/stellaops/authority/plugins" + +# Plug-in manifests live in descriptors below; per-plugin settings are stored +# in the configurationDirectory (YAML files). Authority will load any enabled +# plugins and surface their metadata/capabilities to the host. +plugins: + configurationDirectory: "../etc/authority.plugins" + descriptors: + standard: + type: "standard" + assemblyName: "StellaOps.Authority.Plugin.Standard" + enabled: true + configFile: "standard.yaml" + capabilities: + - password + - bootstrap + - clientProvisioning + metadata: + defaultRole: "operators" + # Example for an external identity provider plugin. Leave disabled unless + # the plug-in package exists under PluginBinaries/Authority. + ldap: + type: "ldap" + assemblyName: "StellaOps.Authority.Plugin.Ldap" + enabled: false + configFile: "ldap.yaml" + capabilities: + - password + - mfa + +# CIDR ranges that bypass network-sensitive policies (e.g. on-host cron jobs). +# Keep the list tight: localhost is sufficient for most air-gapped installs. +bypassNetworks: + - "127.0.0.1/32" + - "::1/128" diff --git a/etc/feedser.yaml.sample b/etc/feedser.yaml.sample new file mode 100644 index 00000000..6393a21f --- /dev/null +++ b/etc/feedser.yaml.sample @@ -0,0 +1,65 @@ +# Feedser configuration template for StellaOps deployments. +# Copy to ../etc/feedser.yaml (relative to the web service content root) +# and adjust the values to match your environment. Environment variables +# (prefixed with FEEDSER_) override these settings at runtime. + +storage: + driver: mongo + # Mongo connection string. Use SRV URI or standard connection string. + dsn: "mongodb://feedser:feedser@mongo:27017/feedser?authSource=admin" + # Optional database name; defaults to the name embedded in the DSN or 'feedser'. + database: "feedser" + # Mongo command timeout in seconds. + commandTimeoutSeconds: 30 + +plugins: + # Feedser resolves plug-ins relative to the content root; override as needed. + baseDirectory: ".." + directory: "PluginBinaries" + searchPatterns: + - "StellaOps.Feedser.Plugin.*.dll" + +telemetry: + enabled: true + enableTracing: false + enableMetrics: false + enableLogging: true + minimumLogLevel: "Information" + serviceName: "stellaops-feedser" + # Configure OTLP endpoint when shipping traces/metrics/logs out-of-band. + otlpEndpoint: "" + # Optional headers for OTLP exporters, for example authentication tokens. + otlpHeaders: {} + # Attach additional resource attributes to telemetry exports. + resourceAttributes: + deployment.environment: "local" + # Emit console exporters for local debugging. + exportConsole: true + +authority: + enabled: false + # Temporary rollout flag. When true, Feedser logs anonymous access but does not fail requests + # without tokens. Set to false before 2025-12-31 UTC to enforce authentication fully. + allowAnonymousFallback: true + # Issuer advertised by StellaOps Authority (e.g. https://authority.stella-ops.local). + issuer: "https://authority.stella-ops.local" + # Optional explicit metadata address; defaults to {issuer}/.well-known/openid-configuration. + metadataAddress: "" + requireHttpsMetadata: true + backchannelTimeoutSeconds: 30 + tokenClockSkewSeconds: 60 + audiences: + - "api://feedser" + requiredScopes: + - "feedser.jobs.trigger" + # Outbound credentials Feedser can use to call Authority (client credentials flow). + clientId: "feedser-jobs" + # Prefer storing the secret outside of the config file. Provide either clientSecret or clientSecretFile. + clientSecret: "" + clientSecretFile: "" + clientScopes: + - "feedser.jobs.trigger" + # Networks allowed to bypass authentication (loopback by default for on-host cron jobs). + bypassNetworks: + - "127.0.0.1/32" + - "::1/128" diff --git a/ops/authority/AGENTS.md b/ops/authority/AGENTS.md new file mode 100644 index 00000000..b19cf07d --- /dev/null +++ b/ops/authority/AGENTS.md @@ -0,0 +1,16 @@ +# Authority DevOps Crew + +## Mission +Operate and harden the StellaOps Authority platform in production and air-gapped environments: container images, deployment assets, observability defaults, backup/restore, and runtime key management. + +## Focus Areas +- **Build & Packaging** – Dockerfiles, OCI bundles, offline artefact refresh. +- **Deployment Tooling** – Compose/Kubernetes manifests, secrets bootstrap, upgrade paths. +- **Observability** – Logging defaults, metrics/trace exporters, dashboards, alert policies. +- **Continuity & Security** – Backup/restore guides, key rotation playbooks, revocation propagation. + +## Working Agreements +- Track work in `ops/authority/TASKS.md` (TODO → DOING → DONE/BLOCKED); keep entries dated. +- Validate container changes with the CI pipeline (`ops/authority` GitHub workflow) before marking DONE. +- Update operator documentation in `docs/` together with any behavioural change. +- Coordinate with Authority Core and Security Guild before altering sensitive defaults (rate limits, crypto providers, revocation jobs). diff --git a/ops/authority/Dockerfile b/ops/authority/Dockerfile new file mode 100644 index 00000000..af76ade5 --- /dev/null +++ b/ops/authority/Dockerfile @@ -0,0 +1,38 @@ +# syntax=docker/dockerfile:1.7-labs + +# +# StellaOps Authority – distroless container build +# Produces a minimal image containing the Authority host and its plugins. +# + +ARG SDK_IMAGE=mcr.microsoft.com/dotnet/nightly/sdk:10.0 +ARG RUNTIME_IMAGE=gcr.io/distroless/dotnet/aspnet:latest + +FROM ${SDK_IMAGE} AS build + +WORKDIR /src + +# Restore & publish +COPY . . +RUN dotnet restore src/StellaOps.sln +RUN dotnet publish src/StellaOps.Authority/StellaOps.Authority/StellaOps.Authority.csproj \ + -c Release \ + -o /app/publish \ + /p:UseAppHost=false + +FROM ${RUNTIME_IMAGE} AS runtime + +WORKDIR /app + +ENV ASPNETCORE_URLS=http://0.0.0.0:8080 +ENV STELLAOPS_AUTHORITY__PLUGINDIRECTORIES__0=/app/plugins +ENV STELLAOPS_AUTHORITY__PLUGINS__CONFIGURATIONDIRECTORY=/app/etc/authority.plugins + +COPY --from=build /app/publish ./ + +# Provide writable mount points for configs/keys/plugins +VOLUME ["/app/etc", "/app/plugins", "/app/keys"] + +EXPOSE 8080 + +ENTRYPOINT ["dotnet", "StellaOps.Authority.dll"] diff --git a/ops/authority/README.md b/ops/authority/README.md new file mode 100644 index 00000000..c399401f --- /dev/null +++ b/ops/authority/README.md @@ -0,0 +1,39 @@ +# StellaOps Authority Container Scaffold + +This directory provides a distroless Dockerfile and `docker-compose` sample for bootstrapping the Authority service alongside MongoDB (required) and Redis (optional). + +## Prerequisites + +- Docker Engine 25+ and Compose V2 +- .NET 10 preview SDK (only required when building locally outside of Compose) +- Populated Authority configuration at `etc/authority.yaml` and plugin manifests under `etc/authority.plugins/` + +## Usage + +```bash +# 1. Ensure configuration files exist (copied from etc/authority.yaml.sample, etc/authority.plugins/*.yaml) +# 2. Build and start the stack +docker compose -f ops/authority/docker-compose.authority.yaml up --build +``` + +`authority.yaml` is mounted read-only at `/etc/authority.yaml` inside the container. Plugin manifests are mounted to `/app/etc/authority.plugins`. Update the issuer URL plus any Mongo credentials in the compose file or via an `.env`. + +To run with pre-built images, replace the `build:` block in the compose file with an `image:` reference. + +## Volumes + +- `mongo-data` – persists MongoDB state. +- `redis-data` – optional Redis persistence (enable the service before use). +- `authority-keys` – writable volume for Authority signing keys. + +## Environment overrides + +Key environment variables (mirroring `StellaOpsAuthorityOptions`): + +| Variable | Description | +| --- | --- | +| `STELLAOPS_AUTHORITY__ISSUER` | Public issuer URL advertised by Authority | +| `STELLAOPS_AUTHORITY__PLUGINDIRECTORIES__0` | Primary plugin binaries directory inside the container | +| `STELLAOPS_AUTHORITY__PLUGINS__CONFIGURATIONDIRECTORY` | Path to plugin manifest directory | + +For additional options, see `etc/authority.yaml.sample`. diff --git a/ops/authority/TASKS.md b/ops/authority/TASKS.md new file mode 100644 index 00000000..7c7ef95c --- /dev/null +++ b/ops/authority/TASKS.md @@ -0,0 +1,6 @@ +# Authority DevOps Task Board (UTC 2025-10-10) + +| ID | Status | Owner(s) | Depends on | Description | Exit Criteria | +| ID | Status | Owner(s) | Depends on | Description | Exit Criteria | +|----|--------|----------|------------|-------------|---------------| +| OPS3.KEY-ROTATION | BLOCKED | DevOps Crew, Authority Core | CORE10.JWKS | Implement key rotation tooling + pipeline hook once rotating JWKS lands. Document SOP and secret handling. | ✅ CLI/script rotates keys + updates JWKS; ✅ Pipeline job documented; ✅ docs/ops runbook updated. | diff --git a/ops/authority/docker-compose.authority.yaml b/ops/authority/docker-compose.authority.yaml new file mode 100644 index 00000000..3f9760fd --- /dev/null +++ b/ops/authority/docker-compose.authority.yaml @@ -0,0 +1,58 @@ +version: "3.9" + +services: + authority: + build: + context: ../.. + dockerfile: ops/authority/Dockerfile + image: stellaops-authority:dev + container_name: stellaops-authority + depends_on: + mongo: + condition: service_started + environment: + # Override issuer to match your deployment URL. + STELLAOPS_AUTHORITY__ISSUER: "https://authority.localtest.me" + # Point the Authority host at the Mongo instance defined below. + STELLAOPS_AUTHORITY__PLUGINDIRECTORIES__0: "/app/plugins" + STELLAOPS_AUTHORITY__PLUGINS__CONFIGURATIONDIRECTORY: "/app/etc/authority.plugins" + volumes: + # Mount Authority configuration + plugins (edit etc/authority.yaml before running). + - ../../etc/authority.yaml:/etc/authority.yaml:ro + - ../../etc/authority.plugins:/app/etc/authority.plugins:ro + # Optional: persist plugin binaries or key material outside the container. + - authority-keys:/app/keys + ports: + - "8080:8080" + restart: unless-stopped + + mongo: + image: mongo:7 + container_name: stellaops-authority-mongo + command: ["mongod", "--bind_ip_all"] + environment: + MONGO_INITDB_ROOT_USERNAME: stellaops + MONGO_INITDB_ROOT_PASSWORD: stellaops + volumes: + - mongo-data:/data/db + ports: + - "27017:27017" + restart: unless-stopped + + redis: + image: redis:7-alpine + container_name: stellaops-authority-redis + command: ["redis-server", "--save", "60", "1"] + volumes: + - redis-data:/data + ports: + - "6379:6379" + restart: unless-stopped + # Uncomment to enable if/when Authority consumes Redis. + # deploy: + # replicas: 0 + +volumes: + mongo-data: + redis-data: + authority-keys: diff --git a/scripts/render_docs.py b/scripts/render_docs.py new file mode 100644 index 00000000..58d27a72 --- /dev/null +++ b/scripts/render_docs.py @@ -0,0 +1,254 @@ +#!/usr/bin/env python3 +"""Render Markdown documentation under docs/ into a static HTML bundle. + +The script converts every Markdown file into a standalone HTML document, +mirroring the original folder structure under the output directory. A +`manifest.json` file is also produced to list the generated documents and +surface basic metadata (title, source path, output path). + +Usage: + python scripts/render_docs.py --source docs --output build/docs-site + +Dependencies: + pip install markdown pygments +""" + +from __future__ import annotations + +import argparse +import json +import logging +import os +import shutil +from dataclasses import dataclass +from datetime import datetime, timezone +from pathlib import Path +from typing import Iterable, List + +import markdown + +# Enable fenced code blocks, tables, and definition lists. These cover the +# Markdown constructs heavily used across the documentation set. +MD_EXTENSIONS = [ + "fenced_code", + "codehilite", + "tables", + "toc", + "def_list", + "admonition", +] + +HTML_TEMPLATE = """ + + + + + {title} + + + +
+{body} +
+
+

Generated on {generated_at} UTC · Source: {source}

+
+ + +""" + + +@dataclass +class DocEntry: + source: Path + output: Path + title: str + + def to_manifest(self) -> dict[str, str]: + return { + "source": self.source.as_posix(), + "output": self.output.as_posix(), + "title": self.title, + } + + +def discover_markdown_files(source_root: Path) -> Iterable[Path]: + for path in source_root.rglob("*.md"): + if path.is_file(): + yield path + + +def read_title(markdown_text: str, fallback: str) -> str: + for raw_line in markdown_text.splitlines(): + line = raw_line.strip() + if line.startswith("#"): + return line.lstrip("#").strip() or fallback + return fallback + + +def convert_markdown(path: Path, source_root: Path, output_root: Path) -> DocEntry: + relative = path.relative_to(source_root) + output_path = output_root / relative.with_suffix(".html") + output_path.parent.mkdir(parents=True, exist_ok=True) + + text = path.read_text(encoding="utf-8") + html_body = markdown.markdown(text, extensions=MD_EXTENSIONS) + + title = read_title(text, fallback=relative.stem.replace("_", " ")) + generated_at = datetime.now(tz=timezone.utc).strftime("%Y-%m-%d %H:%M:%S") + + output_path.write_text( + HTML_TEMPLATE.format( + title=title, + body=html_body, + generated_at=generated_at, + source=relative.as_posix(), + ), + encoding="utf-8", + ) + + return DocEntry(source=relative, output=output_path.relative_to(output_root), title=title) + + +def copy_static_assets(source_root: Path, output_root: Path) -> None: + for path in source_root.rglob("*"): + if path.is_dir() or path.suffix.lower() == ".md": + # Skip Markdown (already rendered separately). + continue + relative = path.relative_to(source_root) + destination = output_root / relative + destination.parent.mkdir(parents=True, exist_ok=True) + destination.write_bytes(path.read_bytes()) + logging.info("Copied asset %s", relative) + + +def write_manifest(entries: Iterable[DocEntry], output_root: Path) -> None: + manifest_path = output_root / "manifest.json" + manifest = [entry.to_manifest() for entry in entries] + manifest_path.write_text(json.dumps(manifest, indent=2), encoding="utf-8") + logging.info("Wrote manifest with %d entries", len(manifest)) + + +def write_index(entries: List[DocEntry], output_root: Path) -> None: + index_path = output_root / "index.html" + generated_at = datetime.now(tz=timezone.utc).strftime("%Y-%m-%d %H:%M:%S") + + items = "\n".join( + f"
  • {entry.title}" f" · {entry.source.as_posix()}
    • " + for entry in sorted(entries, key=lambda e: e.title.lower()) + ) + + html = f""" + + + + + Stella Ops Documentation Index + + + +

    Stella Ops Documentation

    +

    Generated on {generated_at} UTC

    +
      +{items} +
    + + +""" + index_path.write_text(html, encoding="utf-8") + logging.info("Wrote HTML index with %d entries", len(entries)) + + +def parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser(description="Render documentation bundle") + parser.add_argument("--source", default="docs", type=Path, help="Directory containing Markdown sources") + parser.add_argument("--output", default=Path("build/docs-site"), type=Path, help="Directory for rendered output") + parser.add_argument("--clean", action="store_true", help="Remove the output directory before rendering") + return parser.parse_args() + + +def main() -> int: + logging.basicConfig(level=logging.INFO, format="%(levelname)s %(message)s") + args = parse_args() + + source_root: Path = args.source.resolve() + output_root: Path = args.output.resolve() + + if not source_root.exists(): + logging.error("Source directory %s does not exist", source_root) + return os.EX_NOINPUT + + if args.clean and output_root.exists(): + logging.info("Cleaning existing output directory %s", output_root) + shutil.rmtree(output_root) + + output_root.mkdir(parents=True, exist_ok=True) + + entries: List[DocEntry] = [] + for md_file in discover_markdown_files(source_root): + entry = convert_markdown(md_file, source_root, output_root) + entries.append(entry) + logging.info("Rendered %s -> %s", entry.source, entry.output) + + write_manifest(entries, output_root) + write_index(entries, output_root) + copy_static_assets(source_root, output_root) + + logging.info("Documentation bundle available at %s", output_root) + return os.EX_OK + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/update-model-goldens.ps1 b/scripts/update-model-goldens.ps1 new file mode 100644 index 00000000..e4b5a39a --- /dev/null +++ b/scripts/update-model-goldens.ps1 @@ -0,0 +1,9 @@ +Param( + [Parameter(ValueFromRemainingArguments = $true)] + [string[]] $RestArgs +) + +$Root = Split-Path -Parent $PSScriptRoot +$env:UPDATE_GOLDENS = "1" + +dotnet test (Join-Path $Root "src/StellaOps.Feedser.Models.Tests/StellaOps.Feedser.Models.Tests.csproj") @RestArgs diff --git a/scripts/update-model-goldens.sh b/scripts/update-model-goldens.sh new file mode 100644 index 00000000..e668b616 --- /dev/null +++ b/scripts/update-model-goldens.sh @@ -0,0 +1,8 @@ +#!/usr/bin/env bash +set -euo pipefail + +ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" + +export UPDATE_GOLDENS=1 + +dotnet test "$ROOT_DIR/src/StellaOps.Feedser.Models.Tests/StellaOps.Feedser.Models.Tests.csproj" "$@" diff --git a/src/DEDUP_CONFLICTS_RESOLUTION_ALGO.md b/src/DEDUP_CONFLICTS_RESOLUTION_ALGO.md new file mode 100644 index 00000000..71160900 --- /dev/null +++ b/src/DEDUP_CONFLICTS_RESOLUTION_ALGO.md @@ -0,0 +1,143 @@ +````markdown +# Feedser Vulnerability Conflict Resolution Rules + +This document defines the canonical, deterministic conflict resolution strategy for merging vulnerability data from **NVD**, **GHSA**, and **OSV** in Feedser. + +--- + +## 🧭 Source Precedence + +1. **Primary order:** + `GHSA > NVD > OSV` + + **Rationale:** + GHSA advisories are human-curated and fast to correct; NVD has the broadest CVE coverage; OSV excels in ecosystem-specific precision. + +2. **Freshness override (≥48 h):** + If a **lower-priority** source is **newer by at least 48 hours** for a freshness-sensitive field, its value overrides the higher-priority one. + Always store the decision in a provenance record. + +3. **Merge scope:** + Only merge data referring to the **same CVE ID** or the same GHSA/OSV advisory explicitly mapped to that CVE. + +--- + +## 🧩 Field-Level Precedence + +| Field | Priority | Freshness-Sensitive | Notes | +|-------|-----------|--------------------|-------| +| Title / Summary | GHSA → NVD → OSV | ✅ | Prefer concise structured titles | +| Description | GHSA → NVD → OSV | ✅ | | +| Severity (CVSS) | NVD → GHSA → OSV | ❌ | Keep all under `metrics[]`, mark `canonicalMetric` by order | +| Ecosystem Severity Label | GHSA → OSV | ❌ | Supplemental tag only | +| Affected Packages / Ranges | OSV → GHSA → NVD | ✅ | OSV strongest for SemVer normalization | +| CWE(s) | NVD → GHSA → OSV | ❌ | NVD taxonomy most stable | +| References / Links | Union of all | ✅ | Deduplicate by normalized URL | +| Credits / Acknowledgements | Union of all | ✅ | Sort by role, displayName | +| Published / Modified timestamps | Earliest published / Latest modified | ✅ | | +| EPSS / KEV / Exploit status | Specialized feed only | ❌ | Do not override manually | + +--- + +## ⚖️ Deterministic Tie-Breakers + +If precedence and freshness both tie: + +1. **Source order:** GHSA > NVD > OSV +2. **Lexicographic stability:** Prefer shorter normalized text; if equal, ASCIIbetical +3. **Stable hash of payload:** Lowest hash wins + +Each chosen value must store the merge rationale: + +```json +{ + "provenance": { + "source": "GHSA", + "kind": "merge", + "value": "description", + "decisionReason": "precedence" + } +} +```` + +--- + +## 🧮 Merge Algorithm (Pseudocode) + +```csharp +inputs: records = {ghsa?, nvd?, osv?} +out = new CanonicalVuln(CVE) + +foreach field in CANONICAL_SCHEMA: + candidates = collect(values, source, lastModified) + if freshnessSensitive(field) and newerBy48h(lowerPriority): + pick newest + else: + pick by precedence(field) + if tie: + applyTieBreakers() + out.field = normalize(field, value) + out.provenance[field] = decisionTrail + +out.references = dedupe(union(all.references)) +out.affected = normalizeAndUnion(OSV, GHSA, NVD) +out.metrics = rankAndSetCanonical(NVDv3 → GHSA → OSV → v2) +return out +``` + +--- + +## 🔧 Normalization Rules + +* **SemVer:** + Parse with tolerant builder; normalize `v` prefixes; map comparators (`<=`, `<`, `>=`, `>`); expand OSV events into continuous ranges. + +* **Packages:** + Canonical key = `(ecosystem, packageName, language?)`; maintain aliases (purl, npm, Maven GAV, etc.). + +* **CWE:** + Store both ID and name; validate against current CWE catalog. + +* **CVSS:** + Preserve provided vector and base score; recompute only for validation. + +--- + +## ✅ Output Guarantees + +| Property | Description | +| ---------------- | ------------------------------------------------------------------------------- | +| **Reproducible** | Same input → same canonical output | +| **Auditable** | Provenance stored per field | +| **Complete** | Unions with de-duplication | +| **Composable** | Future layers (KEV, EPSS, vendor advisories) can safely extend precedence rules | + +--- + +## 🧠 Example + +* GHSA summary updated on *2025-10-09* +* NVD last modified *2025-10-05* +* OSV updated *2025-10-10* + +→ **Summary:** OSV wins (freshness override) +→ **CVSS:** NVD v3.1 remains canonical +→ **Affected:** OSV ranges canonical; GHSA aliases merged + +--- + +## 🧰 Optional C# Helper Class + +`StellaOps.Feedser.Core/CanonicalMerger.cs` + +Implements: + +* `FieldPrecedenceMap` +* `FreshnessSensitiveFields` +* `ApplyTieBreakers()` +* `NormalizeAndUnion()` + +Deterministically builds `CanonicalVuln` with full provenance tracking. + +``` +``` diff --git a/src/Directory.Build.props b/src/Directory.Build.props new file mode 100644 index 00000000..1b0a699d --- /dev/null +++ b/src/Directory.Build.props @@ -0,0 +1,33 @@ + + + $(SolutionDir)PluginBinaries + $(MSBuildThisFileDirectory)PluginBinaries + $(SolutionDir)PluginBinaries\Authority + $(MSBuildThisFileDirectory)PluginBinaries\Authority + true + true + true + + + + + false + runtime + + + + + + + + + + + + + + + + + + diff --git a/src/Directory.Build.targets b/src/Directory.Build.targets new file mode 100644 index 00000000..a2e810fb --- /dev/null +++ b/src/Directory.Build.targets @@ -0,0 +1,33 @@ + + + + $(FeedserPluginOutputRoot)\$(MSBuildProjectName) + + + + + + + + + + + + + + + + $(AuthorityPluginOutputRoot)\$(MSBuildProjectName) + + + + + + + + + + + + + diff --git a/src/FASTER_MODELING_AND_NORMALIZATION.md b/src/FASTER_MODELING_AND_NORMALIZATION.md new file mode 100644 index 00000000..404a0310 --- /dev/null +++ b/src/FASTER_MODELING_AND_NORMALIZATION.md @@ -0,0 +1,139 @@ +Here’s a quick, practical idea to make your version-range modeling cleaner and faster to query. + +![A simple diagram showing a Vulnerability doc with an embedded normalizedVersions array next to a pipeline icon labeled “simpler aggregations”.](https://images.unsplash.com/photo-1515879218367-8466d910aaa4?q=80\&w=1470\&auto=format\&fit=crop) + +# Rethinking `SemVerRangeBuilder` + MongoDB + +**Problem (today):** Version normalization rules live as a nested object (and often as a bespoke structure per source). This can force awkward `$objectToArray`, `$map`, and conditional logic in pipelines when you need to: + +* match “is version X affected?” +* flatten ranges for analytics +* de-duplicate across sources + +**Proposal:** Store *normalized version rules as an embedded collection (array of small docs)* instead of a single nested object. + +## Minimal background + +* **SemVer normalization**: converting all source-specific version notations into a single, strict representation (e.g., `>=1.2.3 <2.0.0`, exact pins, wildcards). +* **Embedded collection**: an array of consistently shaped items inside the parent doc—great for `$unwind`-centric analytics and direct matches. + +## Suggested shape + +```json +{ + "_id": "VULN-123", + "packageId": "pkg:npm/lodash", + "source": "NVD", + "normalizedVersions": [ + { + "scheme": "semver", + "type": "range", // "range" | "exact" | "lt" | "lte" | "gt" | "gte" + "min": "1.2.3", // optional + "minInclusive": true, // optional + "max": "2.0.0", // optional + "maxInclusive": false, // optional + "notes": "from GHSA GHSA-xxxx" // traceability + }, + { + "scheme": "semver", + "type": "exact", + "value": "1.5.0" + } + ], + "metadata": { "ingestedAt": "2025-10-10T12:00:00Z" } +} +``` + +### Why this helps + +* **Simpler queries** + + * *Is v affected?* + + ```js + db.vulns.aggregate([ + { $match: { packageId: "pkg:npm/lodash" } }, + { $unwind: "$normalizedVersions" }, + { $match: { + $or: [ + { "normalizedVersions.type": "exact", "normalizedVersions.value": "1.5.0" }, + { "normalizedVersions.type": "range", + "normalizedVersions.min": { $lte: "1.5.0" }, + "normalizedVersions.max": { $gt: "1.5.0" } } + ] + }}, + { $project: { _id: 1 } } + ]) + ``` + * No `$objectToArray`, fewer `$cond`s. + +* **Cheaper storage** + + * Arrays of tiny docs compress well and avoid wide nested structures with many nulls/keys. + +* **Easier dedup/merge** + + * `$unwind` → normalize → `$group` by `{scheme,type,min,max,value}` to collapse equivalent rules across sources. + +## Builder changes (`SemVerRangeBuilder`) + +* **Emit items, not a monolith**: have the builder return `IEnumerable`. +* **Normalize early**: resolve “aliases” (`1.2.x`, `^1.2.3`, distro styles) into canonical `(type,min,max,…)` before persistence. +* **Traceability**: include `notes`/`sourceRef` on each rule so you can re-materialize provenance during audits. + +### C# sketch + +```csharp +public record NormalizedVersionRule( + string Scheme, // "semver" + string Type, // "range" | "exact" | ... + string? Min = null, + bool? MinInclusive = null, + string? Max = null, + bool? MaxInclusive = null, + string? Value = null, + string? Notes = null +); + +public static class SemVerRangeBuilder +{ + public static IEnumerable Build(string raw) + { + // parse raw (^1.2.3, 1.2.x, <=2.0.0, etc.) + // yield canonical rules: + yield return new NormalizedVersionRule( + Scheme: "semver", + Type: "range", + Min: "1.2.3", + MinInclusive: true, + Max: "2.0.0", + MaxInclusive: false, + Notes: "nvd:ABC-123" + ); + } +} +``` + +## Aggregation patterns you unlock + +* **Fast “affected version” lookups** via `$unwind + $match` (can complement with a computed sort key). +* **Rollups**: count of vulns per `(major,minor)` by mapping each rule into bucketed segments. +* **Cross-source reconciliation**: group identical rules to de-duplicate. + +## Indexing tips + +* Compound index on `{ packageId: 1, "normalizedVersions.scheme": 1, "normalizedVersions.type": 1 }`. +* If lookups by exact value are common: add a sparse index on `"normalizedVersions.value"`. + +## Migration path (safe + incremental) + +1. **Dual-write**: keep old nested object while writing the new `normalizedVersions` array. +2. **Backfill** existing docs with a one-time script using your current builder. +3. **Cutover** queries/aggregations to the new path (behind a feature flag). +4. **Clean up** old field after soak. + +If you want, I can draft: + +* a one-time Mongo backfill script, +* the new EF/Mongo C# POCOs, and +* a test matrix (edge cases: prerelease tags, build metadata, `0.*` semantics, distro-style ranges). diff --git a/src/StellaOps.Authority/AGENTS.md b/src/StellaOps.Authority/AGENTS.md new file mode 100644 index 00000000..63d2f8f6 --- /dev/null +++ b/src/StellaOps.Authority/AGENTS.md @@ -0,0 +1,20 @@ +# Authority Host Crew + +## Mission +Own the StellaOps Authority host service: ASP.NET minimal API, OpenIddict flows, plugin loading, storage orchestration, and cross-cutting security controls (rate limiting, audit logging, revocation exports). + +## Teams On Call +- Team 2 (Authority Core) +- Team 8 (Security Guild) — collaborates on security-sensitive endpoints + +## Operating Principles +- Deterministic responses, structured logging, cancellation-ready handlers. +- Use `StellaOps.Cryptography` abstractions for any crypto operations. +- Every change updates `TASKS.md` and related docs/tests. +- Coordinate with plugin teams before altering plugin-facing contracts. + +## Key Directories +- `src/StellaOps.Authority/` — host app +- `src/StellaOps.Authority.Tests/` — integration/unit tests +- `src/StellaOps.Authority.Storage.Mongo/` — data access helpers +- `src/StellaOps.Authority.Plugin.Standard/` — default identity provider plugin diff --git a/src/StellaOps.Authority/StellaOps.Auth.Abstractions.Tests/NetworkMaskMatcherTests.cs b/src/StellaOps.Authority/StellaOps.Auth.Abstractions.Tests/NetworkMaskMatcherTests.cs new file mode 100644 index 00000000..87162e0f --- /dev/null +++ b/src/StellaOps.Authority/StellaOps.Auth.Abstractions.Tests/NetworkMaskMatcherTests.cs @@ -0,0 +1,75 @@ +using System; +using System.Net; +using StellaOps.Auth.Abstractions; +using Xunit; + +namespace StellaOps.Auth.Abstractions.Tests; + +public class NetworkMaskMatcherTests +{ + [Fact] + public void Parse_SingleAddress_YieldsHostMask() + { + var mask = NetworkMask.Parse("192.168.1.42"); + + Assert.Equal(32, mask.PrefixLength); + Assert.True(mask.Contains(IPAddress.Parse("192.168.1.42"))); + Assert.False(mask.Contains(IPAddress.Parse("192.168.1.43"))); + } + + [Fact] + public void Parse_Cidr_NormalisesHostBits() + { + var mask = NetworkMask.Parse("10.0.15.9/20"); + + Assert.Equal("10.0.0.0/20", mask.ToString()); + Assert.True(mask.Contains(IPAddress.Parse("10.0.8.1"))); + Assert.False(mask.Contains(IPAddress.Parse("10.0.32.1"))); + } + + [Fact] + public void Contains_ReturnsFalse_ForMismatchedAddressFamily() + { + var mask = NetworkMask.Parse("192.168.0.0/16"); + + Assert.False(mask.Contains(IPAddress.IPv6Loopback)); + } + + [Fact] + public void Matcher_AllowsAll_WhenStarProvided() + { + var matcher = new NetworkMaskMatcher(new[] { "*" }); + + Assert.False(matcher.IsEmpty); + Assert.True(matcher.IsAllowed(IPAddress.Parse("203.0.113.10"))); + Assert.True(matcher.IsAllowed(IPAddress.IPv6Loopback)); + } + + [Fact] + public void Matcher_ReturnsFalse_WhenNoMasksConfigured() + { + var matcher = new NetworkMaskMatcher(Array.Empty()); + + Assert.True(matcher.IsEmpty); + Assert.False(matcher.IsAllowed(IPAddress.Parse("127.0.0.1"))); + Assert.False(matcher.IsAllowed(null)); + } + + [Fact] + public void Matcher_SupportsIpv4AndIpv6Masks() + { + var matcher = new NetworkMaskMatcher(new[] { "192.168.0.0/24", "::1/128" }); + + Assert.True(matcher.IsAllowed(IPAddress.Parse("192.168.0.42"))); + Assert.False(matcher.IsAllowed(IPAddress.Parse("10.0.0.1"))); + Assert.True(matcher.IsAllowed(IPAddress.IPv6Loopback)); + Assert.False(matcher.IsAllowed(IPAddress.IPv6Any)); + } + + [Fact] + public void Matcher_Throws_ForInvalidEntries() + { + var exception = Assert.Throws(() => new NetworkMaskMatcher(new[] { "invalid-mask" })); + Assert.Contains("invalid-mask", exception.Message, StringComparison.OrdinalIgnoreCase); + } +} diff --git a/src/StellaOps.Authority/StellaOps.Auth.Abstractions.Tests/StellaOps.Auth.Abstractions.Tests.csproj b/src/StellaOps.Authority/StellaOps.Auth.Abstractions.Tests/StellaOps.Auth.Abstractions.Tests.csproj new file mode 100644 index 00000000..d6f57b7b --- /dev/null +++ b/src/StellaOps.Authority/StellaOps.Auth.Abstractions.Tests/StellaOps.Auth.Abstractions.Tests.csproj @@ -0,0 +1,10 @@ + + + net10.0 + enable + enable + + + + + diff --git a/src/StellaOps.Authority/StellaOps.Auth.Abstractions.Tests/StellaOpsPrincipalBuilderTests.cs b/src/StellaOps.Authority/StellaOps.Auth.Abstractions.Tests/StellaOpsPrincipalBuilderTests.cs new file mode 100644 index 00000000..1dd7b2dd --- /dev/null +++ b/src/StellaOps.Authority/StellaOps.Auth.Abstractions.Tests/StellaOpsPrincipalBuilderTests.cs @@ -0,0 +1,74 @@ +using System; +using System.Linq; +using System.Security.Claims; +using StellaOps.Auth.Abstractions; +using Xunit; + +namespace StellaOps.Auth.Abstractions.Tests; + +public class StellaOpsPrincipalBuilderTests +{ + [Fact] + public void NormalizedScopes_AreSortedDeduplicatedLowerCased() + { + var builder = new StellaOpsPrincipalBuilder() + .WithScopes(new[] { "Feedser.Jobs.Trigger", " feedser.jobs.trigger ", "AUTHORITY.USERS.MANAGE" }) + .WithAudiences(new[] { " api://feedser ", "api://cli", "api://feedser" }); + + Assert.Equal( + new[] { "authority.users.manage", "feedser.jobs.trigger" }, + builder.NormalizedScopes); + + Assert.Equal( + new[] { "api://cli", "api://feedser" }, + builder.Audiences); + } + + [Fact] + public void Build_ConstructsClaimsPrincipalWithNormalisedValues() + { + var now = DateTimeOffset.UtcNow; + var builder = new StellaOpsPrincipalBuilder() + .WithSubject(" user-1 ") + .WithClientId(" cli-01 ") + .WithTenant(" default ") + .WithName(" Jane Doe ") + .WithIdentityProvider(" internal ") + .WithSessionId(" session-123 ") + .WithTokenId(Guid.NewGuid().ToString("N")) + .WithAuthenticationMethod("password") + .WithAuthenticationType(" custom ") + .WithScopes(new[] { "Feedser.Jobs.Trigger", "AUTHORITY.USERS.MANAGE" }) + .WithAudience(" api://feedser ") + .WithIssuedAt(now) + .WithExpires(now.AddMinutes(5)) + .AddClaim(" custom ", " value "); + + var principal = builder.Build(); + var identity = Assert.IsType(principal.Identity); + + Assert.Equal("custom", identity.AuthenticationType); + Assert.Equal("Jane Doe", identity.Name); + Assert.Equal("user-1", principal.FindFirstValue(StellaOpsClaimTypes.Subject)); + Assert.Equal("cli-01", principal.FindFirstValue(StellaOpsClaimTypes.ClientId)); + Assert.Equal("default", principal.FindFirstValue(StellaOpsClaimTypes.Tenant)); + Assert.Equal("internal", principal.FindFirstValue(StellaOpsClaimTypes.IdentityProvider)); + Assert.Equal("session-123", principal.FindFirstValue(StellaOpsClaimTypes.SessionId)); + Assert.Equal("value", principal.FindFirstValue("custom")); + + var scopeClaims = principal.Claims.Where(claim => claim.Type == StellaOpsClaimTypes.ScopeItem).Select(claim => claim.Value).ToArray(); + Assert.Equal(new[] { "authority.users.manage", "feedser.jobs.trigger" }, scopeClaims); + + var scopeList = principal.FindFirstValue(StellaOpsClaimTypes.Scope); + Assert.Equal("authority.users.manage feedser.jobs.trigger", scopeList); + + var audienceClaims = principal.Claims.Where(claim => claim.Type == StellaOpsClaimTypes.Audience).Select(claim => claim.Value).ToArray(); + Assert.Equal(new[] { "api://feedser" }, audienceClaims); + + var issuedAt = principal.FindFirstValue("iat"); + Assert.Equal(now.ToUnixTimeSeconds().ToString(), issuedAt); + + var expires = principal.FindFirstValue("exp"); + Assert.Equal(now.AddMinutes(5).ToUnixTimeSeconds().ToString(), expires); + } +} diff --git a/src/StellaOps.Authority/StellaOps.Auth.Abstractions.Tests/StellaOpsProblemResultFactoryTests.cs b/src/StellaOps.Authority/StellaOps.Auth.Abstractions.Tests/StellaOpsProblemResultFactoryTests.cs new file mode 100644 index 00000000..c9243ccc --- /dev/null +++ b/src/StellaOps.Authority/StellaOps.Auth.Abstractions.Tests/StellaOpsProblemResultFactoryTests.cs @@ -0,0 +1,53 @@ +using Microsoft.AspNetCore.Http; +using Microsoft.AspNetCore.Http.HttpResults; +using Microsoft.AspNetCore.Mvc; +using StellaOps.Auth.Abstractions; +using Xunit; + +namespace StellaOps.Auth.Abstractions.Tests; + +public class StellaOpsProblemResultFactoryTests +{ + [Fact] + public void AuthenticationRequired_ReturnsCanonicalProblem() + { + var result = StellaOpsProblemResultFactory.AuthenticationRequired(instance: "/jobs"); + + Assert.Equal(StatusCodes.Status401Unauthorized, result.StatusCode); + var details = Assert.IsType(result.ProblemDetails); + Assert.Equal("https://docs.stella-ops.org/problems/authentication-required", details.Type); + Assert.Equal("Authentication required", details.Title); + Assert.Equal("/jobs", details.Instance); + Assert.Equal("unauthorized", details.Extensions["error"]); + Assert.Equal(details.Detail, details.Extensions["error_description"]); + } + + [Fact] + public void InvalidToken_UsesProvidedDetail() + { + var result = StellaOpsProblemResultFactory.InvalidToken("expired refresh token"); + + var details = Assert.IsType(result.ProblemDetails); + Assert.Equal(StatusCodes.Status401Unauthorized, result.StatusCode); + Assert.Equal("expired refresh token", details.Detail); + Assert.Equal("invalid_token", details.Extensions["error"]); + } + + [Fact] + public void InsufficientScope_AddsScopeExtensions() + { + var result = StellaOpsProblemResultFactory.InsufficientScope( + new[] { StellaOpsScopes.FeedserJobsTrigger }, + new[] { StellaOpsScopes.AuthorityUsersManage }, + instance: "/jobs/trigger"); + + Assert.Equal(StatusCodes.Status403Forbidden, result.StatusCode); + + var details = Assert.IsType(result.ProblemDetails); + Assert.Equal("https://docs.stella-ops.org/problems/insufficient-scope", details.Type); + Assert.Equal("insufficient_scope", details.Extensions["error"]); + Assert.Equal(new[] { StellaOpsScopes.FeedserJobsTrigger }, Assert.IsType(details.Extensions["required_scopes"])); + Assert.Equal(new[] { StellaOpsScopes.AuthorityUsersManage }, Assert.IsType(details.Extensions["granted_scopes"])); + Assert.Equal("/jobs/trigger", details.Instance); + } +} diff --git a/src/StellaOps.Authority/StellaOps.Auth.Abstractions/AuthorityTelemetry.cs b/src/StellaOps.Authority/StellaOps.Auth.Abstractions/AuthorityTelemetry.cs new file mode 100644 index 00000000..f1f8915a --- /dev/null +++ b/src/StellaOps.Authority/StellaOps.Auth.Abstractions/AuthorityTelemetry.cs @@ -0,0 +1,56 @@ +using System; +using System.Collections.Generic; +using System.Reflection; + +namespace StellaOps.Auth; + +/// +/// Canonical telemetry metadata for the StellaOps Authority stack. +/// +public static class AuthorityTelemetry +{ + /// + /// service.name resource attribute recorded by Authority components. + /// + public const string ServiceName = "stellaops-authority"; + + /// + /// service.namespace resource attribute aligning Authority with other StellaOps services. + /// + public const string ServiceNamespace = "stellaops"; + + /// + /// Activity source identifier used by Authority instrumentation. + /// + public const string ActivitySourceName = "StellaOps.Authority"; + + /// + /// Meter name used by Authority instrumentation. + /// + public const string MeterName = "StellaOps.Authority"; + + /// + /// Builds the default set of resource attributes (service name/namespace/version). + /// + /// Optional assembly used to resolve the service version. + public static IReadOnlyDictionary BuildDefaultResourceAttributes(Assembly? assembly = null) + { + var version = ResolveServiceVersion(assembly); + + return new Dictionary(StringComparer.OrdinalIgnoreCase) + { + ["service.name"] = ServiceName, + ["service.namespace"] = ServiceNamespace, + ["service.version"] = version + }; + } + + /// + /// Resolves the service version string from the provided assembly (defaults to the Authority telemetry assembly). + /// + public static string ResolveServiceVersion(Assembly? assembly = null) + { + assembly ??= typeof(AuthorityTelemetry).Assembly; + return assembly.GetName().Version?.ToString() ?? "0.0.0"; + } +} diff --git a/src/StellaOps.Authority/StellaOps.Auth.Abstractions/NetworkMask.cs b/src/StellaOps.Authority/StellaOps.Auth.Abstractions/NetworkMask.cs new file mode 100644 index 00000000..266c60fb --- /dev/null +++ b/src/StellaOps.Authority/StellaOps.Auth.Abstractions/NetworkMask.cs @@ -0,0 +1,181 @@ +using System; +using System.Globalization; +using System.Net; +using System.Net.Sockets; + +namespace StellaOps.Auth.Abstractions; + +/// +/// Represents an IP network expressed in CIDR notation. +/// +public readonly record struct NetworkMask +{ + private readonly IPAddress address; + + /// + /// Initialises a new . + /// + /// Canonical network address with host bits zeroed. + /// Prefix length (0-32 for IPv4, 0-128 for IPv6). + public NetworkMask(IPAddress network, int prefixLength) + { + ArgumentNullException.ThrowIfNull(network); + + var maxPrefix = GetMaxPrefix(network); + if (prefixLength is < 0 or > 128 || prefixLength > maxPrefix) + { + throw new ArgumentOutOfRangeException(nameof(prefixLength), $"Prefix length must be between 0 and {maxPrefix} for {network.AddressFamily}."); + } + + address = Normalize(network, prefixLength); + PrefixLength = prefixLength; + } + + /// + /// Canonical network address with host bits zeroed. + /// + public IPAddress Network => address; + + /// + /// Prefix length. + /// + public int PrefixLength { get; } + + /// + /// Attempts to parse the supplied value as CIDR notation or a single IP address. + /// + /// Thrown when the input is not recognised. + public static NetworkMask Parse(string value) + { + if (!TryParse(value, out var mask)) + { + throw new FormatException($"'{value}' is not a valid CIDR or IP address."); + } + + return mask; + } + + /// + /// Attempts to parse the supplied value as CIDR notation or a single IP address. + /// + public static bool TryParse(string? value, out NetworkMask mask) + { + mask = default; + + if (string.IsNullOrWhiteSpace(value)) + { + return false; + } + + var trimmed = value.Trim(); + var slashIndex = trimmed.IndexOf('/', StringComparison.Ordinal); + + if (slashIndex < 0) + { + if (!IPAddress.TryParse(trimmed, out var singleAddress)) + { + return false; + } + + var defaultPrefix = singleAddress.AddressFamily == AddressFamily.InterNetwork ? 32 : 128; + mask = new NetworkMask(singleAddress, defaultPrefix); + return true; + } + + var addressPart = trimmed[..slashIndex]; + var prefixPart = trimmed[(slashIndex + 1)..]; + + if (!IPAddress.TryParse(addressPart, out var networkAddress)) + { + return false; + } + + if (!int.TryParse(prefixPart, NumberStyles.Integer, CultureInfo.InvariantCulture, out var prefixLength)) + { + return false; + } + + try + { + mask = new NetworkMask(networkAddress, prefixLength); + return true; + } + catch (ArgumentOutOfRangeException) + { + return false; + } + } + + /// + /// Determines whether the provided address belongs to this network. + /// + public bool Contains(IPAddress address) + { + ArgumentNullException.ThrowIfNull(address); + + if (address.AddressFamily != this.address.AddressFamily) + { + return false; + } + + if (PrefixLength == 0) + { + return true; + } + + var targetBytes = address.GetAddressBytes(); + var networkBytes = this.address.GetAddressBytes(); + + var fullBytes = PrefixLength / 8; + for (var i = 0; i < fullBytes; i++) + { + if (targetBytes[i] != networkBytes[i]) + { + return false; + } + } + + var remainder = PrefixLength % 8; + if (remainder == 0) + { + return true; + } + + var mask = (byte)(0xFF << (8 - remainder)); + return (targetBytes[fullBytes] & mask) == networkBytes[fullBytes]; + } + + private static int GetMaxPrefix(IPAddress address) + => address.AddressFamily == AddressFamily.InterNetwork ? 32 : + address.AddressFamily == AddressFamily.InterNetworkV6 ? 128 : + throw new ArgumentOutOfRangeException(nameof(address), $"Unsupported address family {address.AddressFamily}."); + + private static IPAddress Normalize(IPAddress address, int prefixLength) + { + var bytes = address.GetAddressBytes(); + + var fullBytes = prefixLength / 8; + var remainder = prefixLength % 8; + + if (fullBytes < bytes.Length) + { + if (remainder > 0) + { + var mask = (byte)(0xFF << (8 - remainder)); + bytes[fullBytes] &= mask; + fullBytes++; + } + + for (var index = fullBytes; index < bytes.Length; index++) + { + bytes[index] = 0; + } + } + + return new IPAddress(bytes); + } + + /// + public override string ToString() + => $"{Network}/{PrefixLength}"; +} diff --git a/src/StellaOps.Authority/StellaOps.Auth.Abstractions/NetworkMaskMatcher.cs b/src/StellaOps.Authority/StellaOps.Auth.Abstractions/NetworkMaskMatcher.cs new file mode 100644 index 00000000..94bc3757 --- /dev/null +++ b/src/StellaOps.Authority/StellaOps.Auth.Abstractions/NetworkMaskMatcher.cs @@ -0,0 +1,139 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Net; + +namespace StellaOps.Auth.Abstractions; + +/// +/// Evaluates remote addresses against configured network masks. +/// +public sealed class NetworkMaskMatcher +{ + private readonly NetworkMask[] masks; + private readonly bool matchAll; + + /// + /// Creates a matcher from raw CIDR strings. + /// + /// Sequence of CIDR entries or IP addresses. + /// Thrown when a value cannot be parsed. + public NetworkMaskMatcher(IEnumerable? values) + : this(Parse(values)) + { + } + + /// + /// Creates a matcher from already parsed masks. + /// + /// Sequence of network masks. + public NetworkMaskMatcher(IEnumerable masks) + { + ArgumentNullException.ThrowIfNull(masks); + + var unique = new HashSet(); + foreach (var mask in masks) + { + unique.Add(mask); + } + + this.masks = unique.ToArray(); + matchAll = this.masks.Length == 1 && this.masks[0].PrefixLength == 0; + } + + private NetworkMaskMatcher((bool MatchAll, NetworkMask[] Masks) parsed) + { + matchAll = parsed.MatchAll; + masks = parsed.Masks; + } + + /// + /// Gets a matcher that allows every address. + /// + public static NetworkMaskMatcher AllowAll { get; } = new((true, Array.Empty())); + + /// + /// Gets a matcher that denies every address (no masks configured). + /// + public static NetworkMaskMatcher DenyAll { get; } = new((false, Array.Empty())); + + /// + /// Indicates whether this matcher has no masks configured and does not allow all. + /// + public bool IsEmpty => !matchAll && masks.Length == 0; + + /// + /// Returns the configured masks. + /// + public IReadOnlyList Masks => masks; + + /// + /// Checks whether the provided address matches any of the configured masks. + /// + /// Remote address to test. + /// true when the address is allowed. + public bool IsAllowed(IPAddress? address) + { + if (address is null) + { + return false; + } + + if (matchAll) + { + return true; + } + + if (masks.Length == 0) + { + return false; + } + + foreach (var mask in masks) + { + if (mask.Contains(address)) + { + return true; + } + } + + return false; + } + + private static (bool MatchAll, NetworkMask[] Masks) Parse(IEnumerable? values) + { + if (values is null) + { + return (false, Array.Empty()); + } + + var unique = new HashSet(); + + foreach (var raw in values) + { + if (string.IsNullOrWhiteSpace(raw)) + { + continue; + } + + var trimmed = raw.Trim(); + + if (IsAllowAll(trimmed)) + { + return (true, Array.Empty()); + } + + if (!NetworkMask.TryParse(trimmed, out var mask)) + { + throw new FormatException($"'{trimmed}' is not a valid network mask or IP address."); + } + + unique.Add(mask); + } + + return (false, unique.ToArray()); + } + + private static bool IsAllowAll(string value) + => value is "*" or "0.0.0.0/0" or "::/0"; +} diff --git a/src/StellaOps.Authority/StellaOps.Auth.Abstractions/README.NuGet.md b/src/StellaOps.Authority/StellaOps.Auth.Abstractions/README.NuGet.md new file mode 100644 index 00000000..b6d3ea49 --- /dev/null +++ b/src/StellaOps.Authority/StellaOps.Auth.Abstractions/README.NuGet.md @@ -0,0 +1,9 @@ +# StellaOps.Auth.Abstractions + +Shared authentication primitives for StellaOps services: + +- Scope and claim constants aligned with StellaOps Authority. +- Deterministic `PrincipalBuilder` and `ProblemResultFactory` helpers. +- Utility types used by resource servers, plug-ins, and client libraries. + +These abstractions are referenced by `StellaOps.Auth.ServerIntegration` and `StellaOps.Auth.Client`. Review `docs/dev/32_AUTH_CLIENT_GUIDE.md` for downstream integration patterns. diff --git a/src/StellaOps.Authority/StellaOps.Auth.Abstractions/StellaOps.Auth.Abstractions.csproj b/src/StellaOps.Authority/StellaOps.Auth.Abstractions/StellaOps.Auth.Abstractions.csproj new file mode 100644 index 00000000..4150c960 --- /dev/null +++ b/src/StellaOps.Authority/StellaOps.Auth.Abstractions/StellaOps.Auth.Abstractions.csproj @@ -0,0 +1,37 @@ + + + net10.0 + preview + enable + enable + true + + + StellaOps.Auth.Abstractions + Core authority authentication abstractions, scopes, and helpers for StellaOps services. + StellaOps + StellaOps + AGPL-3.0-or-later + https://stella-ops.org + https://git.stella-ops.org/stella-ops.org/git.stella-ops.org + git + true + true + true + snupkg + stellaops;authentication;authority;oauth2 + true + $(NoWarn);1591 + README.NuGet.md + 1.0.0-preview.1 + + + + + + + + + + + diff --git a/src/StellaOps.Authority/StellaOps.Auth.Abstractions/StellaOpsAuthenticationDefaults.cs b/src/StellaOps.Authority/StellaOps.Auth.Abstractions/StellaOpsAuthenticationDefaults.cs new file mode 100644 index 00000000..f8720884 --- /dev/null +++ b/src/StellaOps.Authority/StellaOps.Auth.Abstractions/StellaOpsAuthenticationDefaults.cs @@ -0,0 +1,22 @@ +namespace StellaOps.Auth.Abstractions; + +/// +/// Default authentication constants used by StellaOps resource servers and clients. +/// +public static class StellaOpsAuthenticationDefaults +{ + /// + /// Default authentication scheme for StellaOps bearer tokens. + /// + public const string AuthenticationScheme = "StellaOpsBearer"; + + /// + /// Logical authentication type attached to . + /// + public const string AuthenticationType = "StellaOps"; + + /// + /// Policy prefix applied to named authorization policies. + /// + public const string PolicyPrefix = "StellaOps.Policy."; +} diff --git a/src/StellaOps.Authority/StellaOps.Auth.Abstractions/StellaOpsClaimTypes.cs b/src/StellaOps.Authority/StellaOps.Auth.Abstractions/StellaOpsClaimTypes.cs new file mode 100644 index 00000000..54e4174b --- /dev/null +++ b/src/StellaOps.Authority/StellaOps.Auth.Abstractions/StellaOpsClaimTypes.cs @@ -0,0 +1,57 @@ +namespace StellaOps.Auth.Abstractions; + +/// +/// Canonical claim type identifiers used across StellaOps services. +/// +public static class StellaOpsClaimTypes +{ + /// + /// Subject identifier claim (maps to sub in JWTs). + /// + public const string Subject = "sub"; + + /// + /// StellaOps tenant identifier claim (multi-tenant deployments). + /// + public const string Tenant = "stellaops:tenant"; + + /// + /// OAuth2/OIDC client identifier claim (maps to client_id). + /// + public const string ClientId = "client_id"; + + /// + /// Unique token identifier claim (maps to jti). + /// + public const string TokenId = "jti"; + + /// + /// Authentication method reference claim (amr). + /// + public const string AuthenticationMethod = "amr"; + + /// + /// Space separated scope list (scope). + /// + public const string Scope = "scope"; + + /// + /// Individual scope items (scp). + /// + public const string ScopeItem = "scp"; + + /// + /// OAuth2 resource audiences (aud). + /// + public const string Audience = "aud"; + + /// + /// Identity provider hint for downstream services. + /// + public const string IdentityProvider = "stellaops:idp"; + + /// + /// Session identifier claim (sid). + /// + public const string SessionId = "sid"; +} diff --git a/src/StellaOps.Authority/StellaOps.Auth.Abstractions/StellaOpsPrincipalBuilder.cs b/src/StellaOps.Authority/StellaOps.Auth.Abstractions/StellaOpsPrincipalBuilder.cs new file mode 100644 index 00000000..9ff3fc32 --- /dev/null +++ b/src/StellaOps.Authority/StellaOps.Auth.Abstractions/StellaOpsPrincipalBuilder.cs @@ -0,0 +1,287 @@ +using System; +using System.Collections.Generic; +using System.Globalization; +using System.Linq; +using System.Security.Claims; + +namespace StellaOps.Auth.Abstractions; + +/// +/// Fluent helper used to construct instances that follow StellaOps conventions. +/// +public sealed class StellaOpsPrincipalBuilder +{ + private readonly Dictionary singleClaims = new(StringComparer.Ordinal); + private readonly List additionalClaims = new(); + private readonly HashSet scopes = new(StringComparer.OrdinalIgnoreCase); + private readonly HashSet audiences = new(StringComparer.OrdinalIgnoreCase); + + private string authenticationType = StellaOpsAuthenticationDefaults.AuthenticationType; + private string nameClaimType = ClaimTypes.Name; + private string roleClaimType = ClaimTypes.Role; + + private string[]? cachedScopes; + private string[]? cachedAudiences; + + /// + /// Adds or replaces the canonical subject identifier. + /// + public StellaOpsPrincipalBuilder WithSubject(string subject) + => SetSingleClaim(StellaOpsClaimTypes.Subject, subject); + + /// + /// Adds or replaces the canonical client identifier. + /// + public StellaOpsPrincipalBuilder WithClientId(string clientId) + => SetSingleClaim(StellaOpsClaimTypes.ClientId, clientId); + + /// + /// Adds or replaces the tenant identifier claim. + /// + public StellaOpsPrincipalBuilder WithTenant(string tenant) + => SetSingleClaim(StellaOpsClaimTypes.Tenant, tenant); + + /// + /// Adds or replaces the user display name claim. + /// + public StellaOpsPrincipalBuilder WithName(string name) + { + ArgumentException.ThrowIfNullOrWhiteSpace(name); + singleClaims[nameClaimType] = new Claim(nameClaimType, name.Trim(), ClaimValueTypes.String); + return this; + } + + /// + /// Adds or replaces the identity provider claim. + /// + public StellaOpsPrincipalBuilder WithIdentityProvider(string identityProvider) + => SetSingleClaim(StellaOpsClaimTypes.IdentityProvider, identityProvider); + + /// + /// Adds or replaces the session identifier claim. + /// + public StellaOpsPrincipalBuilder WithSessionId(string sessionId) + => SetSingleClaim(StellaOpsClaimTypes.SessionId, sessionId); + + /// + /// Adds or replaces the token identifier claim. + /// + public StellaOpsPrincipalBuilder WithTokenId(string tokenId) + => SetSingleClaim(StellaOpsClaimTypes.TokenId, tokenId); + + /// + /// Adds or replaces the authentication method reference claim. + /// + public StellaOpsPrincipalBuilder WithAuthenticationMethod(string method) + => SetSingleClaim(StellaOpsClaimTypes.AuthenticationMethod, method); + + /// + /// Sets the name claim type appended when building the . + /// + public StellaOpsPrincipalBuilder WithNameClaimType(string claimType) + { + ArgumentException.ThrowIfNullOrWhiteSpace(claimType); + nameClaimType = claimType.Trim(); + return this; + } + + /// + /// Sets the role claim type appended when building the . + /// + public StellaOpsPrincipalBuilder WithRoleClaimType(string claimType) + { + ArgumentException.ThrowIfNullOrWhiteSpace(claimType); + roleClaimType = claimType.Trim(); + return this; + } + + /// + /// Sets the authentication type stamped on the . + /// + public StellaOpsPrincipalBuilder WithAuthenticationType(string authenticationType) + { + ArgumentException.ThrowIfNullOrWhiteSpace(authenticationType); + this.authenticationType = authenticationType.Trim(); + return this; + } + + /// + /// Registers the supplied scopes (normalised to lower-case, deduplicated, sorted). + /// + public StellaOpsPrincipalBuilder WithScopes(IEnumerable scopes) + { + ArgumentNullException.ThrowIfNull(scopes); + + foreach (var scope in scopes) + { + var normalized = StellaOpsScopes.Normalize(scope); + if (normalized is null) + { + continue; + } + + if (this.scopes.Add(normalized)) + { + cachedScopes = null; + } + } + + return this; + } + + /// + /// Registers the supplied audiences (trimmed, deduplicated, sorted). + /// + public StellaOpsPrincipalBuilder WithAudiences(IEnumerable audiences) + { + ArgumentNullException.ThrowIfNull(audiences); + + foreach (var audience in audiences) + { + if (string.IsNullOrWhiteSpace(audience)) + { + continue; + } + + if (this.audiences.Add(audience.Trim())) + { + cachedAudiences = null; + } + } + + return this; + } + + /// + /// Adds a single audience. + /// + public StellaOpsPrincipalBuilder WithAudience(string audience) + => WithAudiences(new[] { audience }); + + /// + /// Adds an arbitrary claim (no deduplication is performed). + /// + public StellaOpsPrincipalBuilder AddClaim(string type, string value, string valueType = ClaimValueTypes.String) + { + ArgumentException.ThrowIfNullOrWhiteSpace(type); + ArgumentException.ThrowIfNullOrWhiteSpace(value); + + var trimmedType = type.Trim(); + var trimmedValue = value.Trim(); + + additionalClaims.Add(new Claim(trimmedType, trimmedValue, valueType)); + return this; + } + + /// + /// Adds multiple claims (incoming claims are cloned to enforce value trimming). + /// + public StellaOpsPrincipalBuilder AddClaims(IEnumerable claims) + { + ArgumentNullException.ThrowIfNull(claims); + + foreach (var claim in claims) + { + ArgumentNullException.ThrowIfNull(claim); + AddClaim(claim.Type, claim.Value, claim.ValueType); + } + + return this; + } + + /// + /// Adds an iat (issued at) claim using Unix time seconds. + /// + public StellaOpsPrincipalBuilder WithIssuedAt(DateTimeOffset issuedAt) + => SetSingleClaim("iat", ToUnixTime(issuedAt)); + + /// + /// Adds an nbf (not before) claim using Unix time seconds. + /// + public StellaOpsPrincipalBuilder WithNotBefore(DateTimeOffset notBefore) + => SetSingleClaim("nbf", ToUnixTime(notBefore)); + + /// + /// Adds an exp (expires) claim using Unix time seconds. + /// + public StellaOpsPrincipalBuilder WithExpires(DateTimeOffset expires) + => SetSingleClaim("exp", ToUnixTime(expires)); + + /// + /// Returns the normalised scope list (deduplicated + sorted). + /// + public IReadOnlyCollection NormalizedScopes + { + get + { + cachedScopes ??= scopes.Count == 0 + ? Array.Empty() + : scopes.OrderBy(static scope => scope, StringComparer.Ordinal).ToArray(); + + return cachedScopes; + } + } + + /// + /// Returns the normalised audience list (deduplicated + sorted). + /// + public IReadOnlyCollection Audiences + { + get + { + cachedAudiences ??= audiences.Count == 0 + ? Array.Empty() + : audiences.OrderBy(static audience => audience, StringComparer.Ordinal).ToArray(); + + return cachedAudiences; + } + } + + /// + /// Builds the immutable instance based on the registered data. + /// + public ClaimsPrincipal Build() + { + var claims = new List( + singleClaims.Count + + additionalClaims.Count + + NormalizedScopes.Count * 2 + + Audiences.Count); + + claims.AddRange(singleClaims.Values); + claims.AddRange(additionalClaims); + + if (NormalizedScopes.Count > 0) + { + var joined = string.Join(' ', NormalizedScopes); + claims.Add(new Claim(StellaOpsClaimTypes.Scope, joined, ClaimValueTypes.String)); + + foreach (var scope in NormalizedScopes) + { + claims.Add(new Claim(StellaOpsClaimTypes.ScopeItem, scope, ClaimValueTypes.String)); + } + } + + if (Audiences.Count > 0) + { + foreach (var audience in Audiences) + { + claims.Add(new Claim(StellaOpsClaimTypes.Audience, audience, ClaimValueTypes.String)); + } + } + + var identity = new ClaimsIdentity(claims, authenticationType, nameClaimType, roleClaimType); + return new ClaimsPrincipal(identity); + } + + private StellaOpsPrincipalBuilder SetSingleClaim(string type, string value) + { + ArgumentException.ThrowIfNullOrWhiteSpace(value); + var trimmedValue = value.Trim(); + singleClaims[type] = new Claim(type, trimmedValue, ClaimValueTypes.String); + return this; + } + + private static string ToUnixTime(DateTimeOffset value) + => value.ToUnixTimeSeconds().ToString(CultureInfo.InvariantCulture); +} diff --git a/src/StellaOps.Authority/StellaOps.Auth.Abstractions/StellaOpsProblemResultFactory.cs b/src/StellaOps.Authority/StellaOps.Auth.Abstractions/StellaOpsProblemResultFactory.cs new file mode 100644 index 00000000..f69d36cd --- /dev/null +++ b/src/StellaOps.Authority/StellaOps.Auth.Abstractions/StellaOpsProblemResultFactory.cs @@ -0,0 +1,114 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using Microsoft.AspNetCore.Http; +using Microsoft.AspNetCore.Http.HttpResults; +using Microsoft.AspNetCore.Mvc; + +namespace StellaOps.Auth.Abstractions; + +/// +/// Factory helpers for returning RFC 7807 problem responses using StellaOps conventions. +/// +public static class StellaOpsProblemResultFactory +{ + private const string ProblemBase = "https://docs.stella-ops.org/problems"; + + /// + /// Produces a 401 problem response indicating authentication is required. + /// + public static ProblemHttpResult AuthenticationRequired(string? detail = null, string? instance = null) + => Create( + StatusCodes.Status401Unauthorized, + $"{ProblemBase}/authentication-required", + "Authentication required", + detail ?? "Authentication is required to access this resource.", + instance, + "unauthorized"); + + /// + /// Produces a 401 problem response for invalid, expired, or revoked tokens. + /// + public static ProblemHttpResult InvalidToken(string? detail = null, string? instance = null) + => Create( + StatusCodes.Status401Unauthorized, + $"{ProblemBase}/invalid-token", + "Invalid token", + detail ?? "The supplied access token is invalid, expired, or revoked.", + instance, + "invalid_token"); + + /// + /// Produces a 403 problem response when access is denied. + /// + public static ProblemHttpResult Forbidden(string? detail = null, string? instance = null) + => Create( + StatusCodes.Status403Forbidden, + $"{ProblemBase}/forbidden", + "Forbidden", + detail ?? "The authenticated principal is not authorised to access this resource.", + instance, + "forbidden"); + + /// + /// Produces a 403 problem response for insufficient scopes. + /// + public static ProblemHttpResult InsufficientScope( + IReadOnlyCollection requiredScopes, + IReadOnlyCollection? grantedScopes = null, + string? instance = null) + { + ArgumentNullException.ThrowIfNull(requiredScopes); + + var extensions = new Dictionary(StringComparer.OrdinalIgnoreCase) + { + ["required_scopes"] = requiredScopes.ToArray() + }; + + if (grantedScopes is not null) + { + extensions["granted_scopes"] = grantedScopes.ToArray(); + } + + return Create( + StatusCodes.Status403Forbidden, + $"{ProblemBase}/insufficient-scope", + "Insufficient scope", + "The authenticated principal does not hold the scopes required by this resource.", + instance, + "insufficient_scope", + extensions); + } + + private static ProblemHttpResult Create( + int status, + string type, + string title, + string detail, + string? instance, + string error, + IReadOnlyDictionary? extensions = null) + { + var problem = new ProblemDetails + { + Status = status, + Type = type, + Title = title, + Detail = detail, + Instance = instance + }; + + problem.Extensions["error"] = error; + problem.Extensions["error_description"] = detail; + + if (extensions is not null) + { + foreach (var entry in extensions) + { + problem.Extensions[entry.Key] = entry.Value; + } + } + + return TypedResults.Problem(problem); + } +} diff --git a/src/StellaOps.Authority/StellaOps.Auth.Abstractions/StellaOpsScopes.cs b/src/StellaOps.Authority/StellaOps.Auth.Abstractions/StellaOpsScopes.cs new file mode 100644 index 00000000..42587835 --- /dev/null +++ b/src/StellaOps.Authority/StellaOps.Auth.Abstractions/StellaOpsScopes.cs @@ -0,0 +1,79 @@ +using System; +using System.Collections.Generic; + +namespace StellaOps.Auth.Abstractions; + +/// +/// Canonical scope names supported by StellaOps services. +/// +public static class StellaOpsScopes +{ + /// + /// Scope required to trigger Feedser jobs. + /// + public const string FeedserJobsTrigger = "feedser.jobs.trigger"; + + /// + /// Scope required to manage Feedser merge operations. + /// + public const string FeedserMerge = "feedser.merge"; + + /// + /// Scope granting administrative access to Authority user management. + /// + public const string AuthorityUsersManage = "authority.users.manage"; + + /// + /// Scope granting administrative access to Authority client registrations. + /// + public const string AuthorityClientsManage = "authority.clients.manage"; + + /// + /// Scope granting read-only access to Authority audit logs. + /// + public const string AuthorityAuditRead = "authority.audit.read"; + + /// + /// Synthetic scope representing trusted network bypass. + /// + public const string Bypass = "stellaops.bypass"; + + private static readonly HashSet KnownScopes = new(StringComparer.OrdinalIgnoreCase) + { + FeedserJobsTrigger, + FeedserMerge, + AuthorityUsersManage, + AuthorityClientsManage, + AuthorityAuditRead, + Bypass + }; + + /// + /// Normalises a scope string (trim/convert to lower case). + /// + /// Scope raw value. + /// Normalised scope or null when the input is blank. + public static string? Normalize(string? scope) + { + if (string.IsNullOrWhiteSpace(scope)) + { + return null; + } + + return scope.Trim().ToLowerInvariant(); + } + + /// + /// Checks whether the provided scope is registered as a built-in StellaOps scope. + /// + public static bool IsKnown(string scope) + { + ArgumentNullException.ThrowIfNull(scope); + return KnownScopes.Contains(scope); + } + + /// + /// Returns the full set of built-in scopes. + /// + public static IReadOnlyCollection All => KnownScopes; +} diff --git a/src/StellaOps.Authority/StellaOps.Auth.Client.Tests/ServiceCollectionExtensionsTests.cs b/src/StellaOps.Authority/StellaOps.Auth.Client.Tests/ServiceCollectionExtensionsTests.cs new file mode 100644 index 00000000..521f4598 --- /dev/null +++ b/src/StellaOps.Authority/StellaOps.Auth.Client.Tests/ServiceCollectionExtensionsTests.cs @@ -0,0 +1,95 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Net; +using System.Net.Http; +using System.Net.Http.Headers; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Http; +using StellaOps.Auth.Client; +using Xunit; + +namespace StellaOps.Auth.Client.Tests; + +public class ServiceCollectionExtensionsTests +{ + [Fact] + public async Task AddStellaOpsAuthClient_ConfiguresRetryPolicy() + { + var services = new ServiceCollection(); + services.AddLogging(); + + services.AddStellaOpsAuthClient(options => + { + options.Authority = "https://authority.test"; + options.RetryDelays.Clear(); + options.RetryDelays.Add(TimeSpan.FromMilliseconds(1)); + options.DiscoveryCacheLifetime = TimeSpan.FromMinutes(1); + options.JwksCacheLifetime = TimeSpan.FromMinutes(1); + options.AllowOfflineCacheFallback = false; + }); + + var recordedHandlers = new List(); + var attemptCount = 0; + + services.AddHttpClient() + .ConfigureHttpMessageHandlerBuilder(builder => + { + recordedHandlers = new List(builder.AdditionalHandlers); + + var responses = new Queue>(new[] + { + () => CreateResponse(HttpStatusCode.InternalServerError, "{}"), + () => CreateResponse(HttpStatusCode.OK, "{\"token_endpoint\":\"https://authority.test/connect/token\",\"jwks_uri\":\"https://authority.test/jwks\"}") + }); + + builder.PrimaryHandler = new LambdaHttpMessageHandler((_, _) => + { + attemptCount++; + + if (responses.Count == 0) + { + return Task.FromResult(CreateResponse(HttpStatusCode.OK, "{}")); + } + + var factory = responses.Dequeue(); + return Task.FromResult(factory()); + }); + }); + + using var provider = services.BuildServiceProvider(); + + var cache = provider.GetRequiredService(); + var configuration = await cache.GetAsync(CancellationToken.None); + + Assert.Equal(new Uri("https://authority.test/connect/token"), configuration.TokenEndpoint); + Assert.Equal(2, attemptCount); + Assert.NotEmpty(recordedHandlers); + Assert.Contains(recordedHandlers, handler => handler.GetType().Name.Contains("PolicyHttpMessageHandler", StringComparison.Ordinal)); + } + + private static HttpResponseMessage CreateResponse(HttpStatusCode statusCode, string jsonContent) + { + return new HttpResponseMessage(statusCode) + { + Content = new StringContent(jsonContent) + { + Headers = { ContentType = new MediaTypeHeaderValue("application/json") } + } + }; + } + private sealed class LambdaHttpMessageHandler : HttpMessageHandler + { + private readonly Func> responder; + + public LambdaHttpMessageHandler(Func> responder) + { + this.responder = responder; + } + + protected override Task SendAsync(HttpRequestMessage request, CancellationToken cancellationToken) + => responder(request, cancellationToken); + } +} diff --git a/src/StellaOps.Authority/StellaOps.Auth.Client.Tests/StellaOps.Auth.Client.Tests.csproj b/src/StellaOps.Authority/StellaOps.Auth.Client.Tests/StellaOps.Auth.Client.Tests.csproj new file mode 100644 index 00000000..f52435fc --- /dev/null +++ b/src/StellaOps.Authority/StellaOps.Auth.Client.Tests/StellaOps.Auth.Client.Tests.csproj @@ -0,0 +1,15 @@ + + + net10.0 + enable + enable + + + + + + + + + + diff --git a/src/StellaOps.Authority/StellaOps.Auth.Client.Tests/StellaOpsAuthClientOptionsTests.cs b/src/StellaOps.Authority/StellaOps.Auth.Client.Tests/StellaOpsAuthClientOptionsTests.cs new file mode 100644 index 00000000..e49deb14 --- /dev/null +++ b/src/StellaOps.Authority/StellaOps.Auth.Client.Tests/StellaOpsAuthClientOptionsTests.cs @@ -0,0 +1,84 @@ +using System; +using StellaOps.Auth.Client; +using Xunit; + +namespace StellaOps.Auth.Client.Tests; + +public class StellaOpsAuthClientOptionsTests +{ + [Fact] + public void Validate_NormalizesScopes() + { + var options = new StellaOpsAuthClientOptions + { + Authority = "https://authority.test", + ClientId = "cli", + HttpTimeout = TimeSpan.FromSeconds(15) + }; + options.DefaultScopes.Add(" Feedser.Jobs.Trigger "); + options.DefaultScopes.Add("feedser.jobs.trigger"); + options.DefaultScopes.Add("AUTHORITY.USERS.MANAGE"); + + options.Validate(); + + Assert.Equal(new[] { "authority.users.manage", "feedser.jobs.trigger" }, options.NormalizedScopes); + Assert.Equal(new Uri("https://authority.test"), options.AuthorityUri); + Assert.Equal(options.RetryDelays, options.NormalizedRetryDelays); + } + + [Fact] + public void Validate_Throws_When_AuthorityMissing() + { + var options = new StellaOpsAuthClientOptions(); + + var exception = Assert.Throws(() => options.Validate()); + + Assert.Contains("Authority", exception.Message, StringComparison.OrdinalIgnoreCase); + } + + [Fact] + public void Validate_NormalizesRetryDelays() + { + var options = new StellaOpsAuthClientOptions + { + Authority = "https://authority.test" + }; + options.RetryDelays.Clear(); + options.RetryDelays.Add(TimeSpan.Zero); + options.RetryDelays.Add(TimeSpan.FromSeconds(3)); + options.RetryDelays.Add(TimeSpan.FromMilliseconds(-1)); + + options.Validate(); + + Assert.Equal(new[] { TimeSpan.FromSeconds(3) }, options.NormalizedRetryDelays); + Assert.Equal(options.NormalizedRetryDelays, options.RetryDelays); + } + + [Fact] + public void Validate_DisabledRetries_ProducesEmptyDelays() + { + var options = new StellaOpsAuthClientOptions + { + Authority = "https://authority.test", + EnableRetries = false + }; + + options.Validate(); + + Assert.Empty(options.NormalizedRetryDelays); + } + + [Fact] + public void Validate_Throws_When_OfflineToleranceNegative() + { + var options = new StellaOpsAuthClientOptions + { + Authority = "https://authority.test", + OfflineCacheTolerance = TimeSpan.FromSeconds(-1) + }; + + var exception = Assert.Throws(() => options.Validate()); + + Assert.Contains("Offline cache tolerance", exception.Message, StringComparison.OrdinalIgnoreCase); + } +} diff --git a/src/StellaOps.Authority/StellaOps.Auth.Client.Tests/StellaOpsDiscoveryCacheTests.cs b/src/StellaOps.Authority/StellaOps.Auth.Client.Tests/StellaOpsDiscoveryCacheTests.cs new file mode 100644 index 00000000..72e8fafd --- /dev/null +++ b/src/StellaOps.Authority/StellaOps.Auth.Client.Tests/StellaOpsDiscoveryCacheTests.cs @@ -0,0 +1,134 @@ +using System; +using System.Net; +using System.Net.Http; +using System.Net.Http.Headers; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.Extensions.Logging.Abstractions; +using Microsoft.Extensions.Options; +using Microsoft.Extensions.Time.Testing; +using StellaOps.Auth.Client; +using Xunit; + +namespace StellaOps.Auth.Client.Tests; + +public class StellaOpsDiscoveryCacheTests +{ + [Fact] + public async Task GetAsync_UsesOfflineFallbackWithinTolerance() + { + var timeProvider = new FakeTimeProvider(DateTimeOffset.Parse("2025-01-01T00:00:00Z")); + var callCount = 0; + var handler = new StubHttpMessageHandler((request, _) => + { + callCount++; + + if (callCount == 1) + { + return Task.FromResult(CreateJsonResponse("{\"token_endpoint\":\"https://authority.test/connect/token\",\"jwks_uri\":\"https://authority.test/jwks\"}")); + } + + throw new HttpRequestException("offline"); + }); + + var httpClient = new HttpClient(handler); + + var options = new StellaOpsAuthClientOptions + { + Authority = "https://authority.test", + DiscoveryCacheLifetime = TimeSpan.FromMinutes(1), + OfflineCacheTolerance = TimeSpan.FromMinutes(5), + AllowOfflineCacheFallback = true + }; + options.Validate(); + + var monitor = new TestOptionsMonitor(options); + var cache = new StellaOpsDiscoveryCache(httpClient, monitor, timeProvider, NullLogger.Instance); + + var configuration = await cache.GetAsync(CancellationToken.None); + Assert.Equal(new Uri("https://authority.test/connect/token"), configuration.TokenEndpoint); + + timeProvider.Advance(TimeSpan.FromMinutes(1) + TimeSpan.FromSeconds(5)); + + configuration = await cache.GetAsync(CancellationToken.None); + Assert.Equal(new Uri("https://authority.test/connect/token"), configuration.TokenEndpoint); + Assert.Equal(2, callCount); + + var offlineExpiry = GetOfflineExpiry(cache); + Assert.True(offlineExpiry > timeProvider.GetUtcNow()); + + timeProvider.Advance(options.OfflineCacheTolerance + TimeSpan.FromSeconds(1)); + + Assert.True(offlineExpiry < timeProvider.GetUtcNow()); + + HttpRequestException? exception = null; + try + { + await cache.GetAsync(CancellationToken.None); + } + catch (HttpRequestException ex) + { + exception = ex; + } + + Assert.NotNull(exception); + Assert.Equal(3, callCount); + } + + private static HttpResponseMessage CreateJsonResponse(string json) + { + return new HttpResponseMessage(HttpStatusCode.OK) + { + Content = new StringContent(json) + { + Headers = { ContentType = new MediaTypeHeaderValue("application/json") } + } + }; + } + + private sealed class StubHttpMessageHandler : HttpMessageHandler + { + private readonly Func> responder; + + public StubHttpMessageHandler(Func> responder) + { + this.responder = responder; + } + + protected override Task SendAsync(HttpRequestMessage request, CancellationToken cancellationToken) + => responder(request, cancellationToken); + } + + private sealed class TestOptionsMonitor : IOptionsMonitor + where T : class + { + private readonly T value; + + public TestOptionsMonitor(T value) + { + this.value = value; + } + + public T CurrentValue => value; + + public T Get(string? name) => value; + + public IDisposable OnChange(Action listener) => NullDisposable.Instance; + + private sealed class NullDisposable : IDisposable + { + public static NullDisposable Instance { get; } = new(); + + public void Dispose() + { + } + } + } + + private static DateTimeOffset GetOfflineExpiry(StellaOpsDiscoveryCache cache) + { + var field = typeof(StellaOpsDiscoveryCache).GetField("offlineExpiresAt", System.Reflection.BindingFlags.NonPublic | System.Reflection.BindingFlags.Instance); + Assert.NotNull(field); + return (DateTimeOffset)field!.GetValue(cache)!; + } +} diff --git a/src/StellaOps.Authority/StellaOps.Auth.Client.Tests/StellaOpsTokenClientTests.cs b/src/StellaOps.Authority/StellaOps.Auth.Client.Tests/StellaOpsTokenClientTests.cs new file mode 100644 index 00000000..6dbe89ed --- /dev/null +++ b/src/StellaOps.Authority/StellaOps.Auth.Client.Tests/StellaOpsTokenClientTests.cs @@ -0,0 +1,111 @@ +using System; +using System.Collections.Generic; +using System.Net; +using System.Net.Http; +using System.Net.Http.Headers; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.Extensions.Logging.Abstractions; +using Microsoft.Extensions.Options; +using Microsoft.Extensions.Time.Testing; +using StellaOps.Auth.Client; +using Xunit; + +namespace StellaOps.Auth.Client.Tests; + +public class StellaOpsTokenClientTests +{ + [Fact] + public async Task RequestPasswordToken_ReturnsResultAndCaches() + { + var timeProvider = new FakeTimeProvider(DateTimeOffset.Parse("2025-02-01T00:00:00Z")); + var responses = new Queue(); + responses.Enqueue(CreateJsonResponse("{\"token_endpoint\":\"https://authority.test/connect/token\",\"jwks_uri\":\"https://authority.test/jwks\"}")); + responses.Enqueue(CreateJsonResponse("{\"access_token\":\"abc\",\"token_type\":\"Bearer\",\"expires_in\":120,\"scope\":\"feedser.jobs.trigger\"}")); + responses.Enqueue(CreateJsonResponse("{\"keys\":[]}")); + + var handler = new StubHttpMessageHandler((request, cancellationToken) => + { + Assert.True(responses.Count > 0, $"Unexpected request {request.Method} {request.RequestUri}"); + return Task.FromResult(responses.Dequeue()); + }); + + var httpClient = new HttpClient(handler); + + var options = new StellaOpsAuthClientOptions + { + Authority = "https://authority.test", + ClientId = "cli" + }; + options.DefaultScopes.Add("feedser.jobs.trigger"); + options.Validate(); + + var optionsMonitor = new TestOptionsMonitor(options); + var cache = new InMemoryTokenCache(timeProvider, TimeSpan.FromSeconds(5)); + var discoveryCache = new StellaOpsDiscoveryCache(httpClient, optionsMonitor, timeProvider); + var jwksCache = new StellaOpsJwksCache(httpClient, discoveryCache, optionsMonitor, timeProvider); + var client = new StellaOpsTokenClient(httpClient, discoveryCache, jwksCache, optionsMonitor, cache, timeProvider, NullLogger.Instance); + + var result = await client.RequestPasswordTokenAsync("user", "pass"); + + Assert.Equal("abc", result.AccessToken); + Assert.Contains("feedser.jobs.trigger", result.Scopes); + + await client.CacheTokenAsync("key", result.ToCacheEntry()); + var cached = await client.GetCachedTokenAsync("key"); + Assert.NotNull(cached); + Assert.Equal("abc", cached!.AccessToken); + + var jwks = await client.GetJsonWebKeySetAsync(); + Assert.Empty(jwks.Keys); + } + + private static HttpResponseMessage CreateJsonResponse(string json) + { + return new HttpResponseMessage(HttpStatusCode.OK) + { + Content = new StringContent(json) + { + Headers = { ContentType = new MediaTypeHeaderValue("application/json") } + } + }; + } + + private sealed class StubHttpMessageHandler : HttpMessageHandler + { + private readonly Func> responder; + + public StubHttpMessageHandler(Func> responder) + { + this.responder = responder; + } + + protected override Task SendAsync(HttpRequestMessage request, CancellationToken cancellationToken) + => responder(request, cancellationToken); + } + + private sealed class TestOptionsMonitor : IOptionsMonitor + where TOptions : class + { + private readonly TOptions value; + + public TestOptionsMonitor(TOptions value) + { + this.value = value; + } + + public TOptions CurrentValue => value; + + public TOptions Get(string? name) => value; + + public IDisposable OnChange(Action listener) => NullDisposable.Instance; + + private sealed class NullDisposable : IDisposable + { + public static NullDisposable Instance { get; } = new(); + public void Dispose() + { + } + } + } +} diff --git a/src/StellaOps.Authority/StellaOps.Auth.Client.Tests/TokenCacheTests.cs b/src/StellaOps.Authority/StellaOps.Auth.Client.Tests/TokenCacheTests.cs new file mode 100644 index 00000000..09597376 --- /dev/null +++ b/src/StellaOps.Authority/StellaOps.Auth.Client.Tests/TokenCacheTests.cs @@ -0,0 +1,59 @@ +using System; +using System.IO; +using System.Net; +using System.Threading.Tasks; +using Microsoft.Extensions.Time.Testing; +using StellaOps.Auth.Client; +using Xunit; + +namespace StellaOps.Auth.Client.Tests; + +public class TokenCacheTests +{ + [Fact] + public async Task InMemoryTokenCache_ExpiresEntries() + { + var timeProvider = new FakeTimeProvider(DateTimeOffset.Parse("2025-01-01T00:00:00Z")); + var cache = new InMemoryTokenCache(timeProvider, TimeSpan.FromSeconds(5)); + + var entry = new StellaOpsTokenCacheEntry("token", "Bearer", timeProvider.GetUtcNow() + TimeSpan.FromSeconds(10), new[] { "scope" }); + await cache.SetAsync("key", entry); + + var retrieved = await cache.GetAsync("key"); + Assert.NotNull(retrieved); + + timeProvider.Advance(TimeSpan.FromSeconds(12)); + + retrieved = await cache.GetAsync("key"); + Assert.Null(retrieved); + } + + [Fact] + public async Task FileTokenCache_PersistsEntries() + { + var directory = Path.Combine(Path.GetTempPath(), Guid.NewGuid().ToString("N")); + try + { + var timeProvider = new FakeTimeProvider(DateTimeOffset.UtcNow); + var cache = new FileTokenCache(directory, timeProvider, TimeSpan.Zero); + + var entry = new StellaOpsTokenCacheEntry("token", "Bearer", timeProvider.GetUtcNow() + TimeSpan.FromMinutes(5), new[] { "scope" }); + await cache.SetAsync("key", entry); + + var retrieved = await cache.GetAsync("key"); + Assert.NotNull(retrieved); + Assert.Equal("token", retrieved!.AccessToken); + + await cache.RemoveAsync("key"); + retrieved = await cache.GetAsync("key"); + Assert.Null(retrieved); + } + finally + { + if (Directory.Exists(directory)) + { + Directory.Delete(directory, recursive: true); + } + } + } +} diff --git a/src/StellaOps.Authority/StellaOps.Auth.Client/FileTokenCache.cs b/src/StellaOps.Authority/StellaOps.Auth.Client/FileTokenCache.cs new file mode 100644 index 00000000..159cfcff --- /dev/null +++ b/src/StellaOps.Authority/StellaOps.Auth.Client/FileTokenCache.cs @@ -0,0 +1,122 @@ +using System; +using System.IO; +using System.Security.Cryptography; +using System.Text.Json; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.Extensions.Logging; + +namespace StellaOps.Auth.Client; + +/// +/// File-based token cache suitable for CLI/offline usage. +/// +public sealed class FileTokenCache : IStellaOpsTokenCache +{ + private readonly string cacheDirectory; + private readonly TimeProvider timeProvider; + private readonly TimeSpan expirationSkew; + private readonly ILogger? logger; + private readonly JsonSerializerOptions serializerOptions = new(JsonSerializerDefaults.Web) + { + WriteIndented = false + }; + + public FileTokenCache(string cacheDirectory, TimeProvider? timeProvider = null, TimeSpan? expirationSkew = null, ILogger? logger = null) + { + ArgumentException.ThrowIfNullOrWhiteSpace(cacheDirectory); + + this.cacheDirectory = cacheDirectory; + this.timeProvider = timeProvider ?? TimeProvider.System; + this.expirationSkew = expirationSkew ?? TimeSpan.FromSeconds(30); + this.logger = logger; + } + + public async ValueTask GetAsync(string key, CancellationToken cancellationToken = default) + { + ArgumentException.ThrowIfNullOrWhiteSpace(key); + + var path = GetPath(key); + if (!File.Exists(path)) + { + return null; + } + + try + { + await using var stream = new FileStream(path, FileMode.Open, FileAccess.Read, FileShare.Read, 4096, FileOptions.Asynchronous); + var entry = await JsonSerializer.DeserializeAsync(stream, serializerOptions, cancellationToken).ConfigureAwait(false); + + if (entry is null) + { + return null; + } + + entry = entry.NormalizeScopes(); + + if (entry.IsExpired(timeProvider, expirationSkew)) + { + await RemoveInternalAsync(path).ConfigureAwait(false); + return null; + } + + return entry; + } + catch (Exception ex) when (ex is IOException or JsonException or UnauthorizedAccessException) + { + logger?.LogWarning(ex, "Failed to read token cache entry '{CacheKey}'.", key); + return null; + } + } + + public async ValueTask SetAsync(string key, StellaOpsTokenCacheEntry entry, CancellationToken cancellationToken = default) + { + ArgumentException.ThrowIfNullOrWhiteSpace(key); + ArgumentNullException.ThrowIfNull(entry); + + Directory.CreateDirectory(cacheDirectory); + + var path = GetPath(key); + var payload = entry.NormalizeScopes(); + + try + { + await using var stream = new FileStream(path, FileMode.Create, FileAccess.Write, FileShare.None, 4096, FileOptions.Asynchronous); + await JsonSerializer.SerializeAsync(stream, payload, serializerOptions, cancellationToken).ConfigureAwait(false); + } + catch (Exception ex) when (ex is IOException or UnauthorizedAccessException) + { + logger?.LogWarning(ex, "Failed to persist token cache entry '{CacheKey}'.", key); + } + } + + public ValueTask RemoveAsync(string key, CancellationToken cancellationToken = default) + { + ArgumentException.ThrowIfNullOrWhiteSpace(key); + var path = GetPath(key); + return new ValueTask(RemoveInternalAsync(path)); + } + + private async Task RemoveInternalAsync(string path) + { + try + { + if (File.Exists(path)) + { + await Task.Run(() => File.Delete(path)).ConfigureAwait(false); + } + } + catch (Exception ex) when (ex is IOException or UnauthorizedAccessException) + { + logger?.LogDebug(ex, "Failed to remove cache file '{Path}'.", path); + } + } + + private string GetPath(string key) + { + using var sha = SHA256.Create(); + var bytes = System.Text.Encoding.UTF8.GetBytes(key); + var hash = Convert.ToHexString(sha.ComputeHash(bytes)); + return Path.Combine(cacheDirectory, $"{hash}.json"); + } +} diff --git a/src/StellaOps.Authority/StellaOps.Auth.Client/IStellaOpsTokenCache.cs b/src/StellaOps.Authority/StellaOps.Auth.Client/IStellaOpsTokenCache.cs new file mode 100644 index 00000000..80cd9329 --- /dev/null +++ b/src/StellaOps.Authority/StellaOps.Auth.Client/IStellaOpsTokenCache.cs @@ -0,0 +1,25 @@ +using System.Threading; +using System.Threading.Tasks; + +namespace StellaOps.Auth.Client; + +/// +/// Abstraction for caching StellaOps tokens. +/// +public interface IStellaOpsTokenCache +{ + /// + /// Retrieves a cached token entry, if present. + /// + ValueTask GetAsync(string key, CancellationToken cancellationToken = default); + + /// + /// Stores or updates a token entry for the specified key. + /// + ValueTask SetAsync(string key, StellaOpsTokenCacheEntry entry, CancellationToken cancellationToken = default); + + /// + /// Removes the cached entry for the specified key. + /// + ValueTask RemoveAsync(string key, CancellationToken cancellationToken = default); +} diff --git a/src/StellaOps.Authority/StellaOps.Auth.Client/IStellaOpsTokenClient.cs b/src/StellaOps.Authority/StellaOps.Auth.Client/IStellaOpsTokenClient.cs new file mode 100644 index 00000000..0d60dbd3 --- /dev/null +++ b/src/StellaOps.Authority/StellaOps.Auth.Client/IStellaOpsTokenClient.cs @@ -0,0 +1,41 @@ +using System.Threading; +using System.Threading.Tasks; +using Microsoft.IdentityModel.Tokens; + +namespace StellaOps.Auth.Client; + +/// +/// Abstraction for requesting tokens from StellaOps Authority. +/// +public interface IStellaOpsTokenClient +{ + /// + /// Requests an access token using the resource owner password credentials flow. + /// + Task RequestPasswordTokenAsync(string username, string password, string? scope = null, CancellationToken cancellationToken = default); + + /// + /// Requests an access token using the client credentials flow. + /// + Task RequestClientCredentialsTokenAsync(string? scope = null, CancellationToken cancellationToken = default); + + /// + /// Retrieves the cached JWKS document. + /// + Task GetJsonWebKeySetAsync(CancellationToken cancellationToken = default); + + /// + /// Retrieves a cached token entry. + /// + ValueTask GetCachedTokenAsync(string key, CancellationToken cancellationToken = default); + + /// + /// Persists a token entry in the cache. + /// + ValueTask CacheTokenAsync(string key, StellaOpsTokenCacheEntry entry, CancellationToken cancellationToken = default); + + /// + /// Removes a cached entry. + /// + ValueTask ClearCachedTokenAsync(string key, CancellationToken cancellationToken = default); +} diff --git a/src/StellaOps.Authority/StellaOps.Auth.Client/InMemoryTokenCache.cs b/src/StellaOps.Authority/StellaOps.Auth.Client/InMemoryTokenCache.cs new file mode 100644 index 00000000..788a82c9 --- /dev/null +++ b/src/StellaOps.Authority/StellaOps.Auth.Client/InMemoryTokenCache.cs @@ -0,0 +1,58 @@ +using System; +using System.Collections.Concurrent; +using System.Threading; +using System.Threading.Tasks; + +namespace StellaOps.Auth.Client; + +/// +/// In-memory token cache suitable for service scenarios. +/// +public sealed class InMemoryTokenCache : IStellaOpsTokenCache +{ + private readonly ConcurrentDictionary entries = new(StringComparer.Ordinal); + private readonly TimeProvider timeProvider; + private readonly Func normalizer; + private readonly TimeSpan expirationSkew; + + public InMemoryTokenCache(TimeProvider? timeProvider = null, TimeSpan? expirationSkew = null) + { + this.timeProvider = timeProvider ?? TimeProvider.System; + this.expirationSkew = expirationSkew ?? TimeSpan.FromSeconds(30); + normalizer = static entry => entry.NormalizeScopes(); + } + + public ValueTask GetAsync(string key, CancellationToken cancellationToken = default) + { + ArgumentException.ThrowIfNullOrWhiteSpace(key); + + if (!entries.TryGetValue(key, out var entry)) + { + return ValueTask.FromResult(null); + } + + if (entry.IsExpired(timeProvider, expirationSkew)) + { + entries.TryRemove(key, out _); + return ValueTask.FromResult(null); + } + + return ValueTask.FromResult(entry); + } + + public ValueTask SetAsync(string key, StellaOpsTokenCacheEntry entry, CancellationToken cancellationToken = default) + { + ArgumentException.ThrowIfNullOrWhiteSpace(key); + ArgumentNullException.ThrowIfNull(entry); + + entries[key] = normalizer(entry); + return ValueTask.CompletedTask; + } + + public ValueTask RemoveAsync(string key, CancellationToken cancellationToken = default) + { + ArgumentException.ThrowIfNullOrWhiteSpace(key); + entries.TryRemove(key, out _); + return ValueTask.CompletedTask; + } +} diff --git a/src/StellaOps.Authority/StellaOps.Auth.Client/README.NuGet.md b/src/StellaOps.Authority/StellaOps.Auth.Client/README.NuGet.md new file mode 100644 index 00000000..3369a62e --- /dev/null +++ b/src/StellaOps.Authority/StellaOps.Auth.Client/README.NuGet.md @@ -0,0 +1,9 @@ +# StellaOps.Auth.Client + +Typed OpenID Connect client used by StellaOps services, agents, and tooling to talk to **StellaOps Authority**. It provides: + +- Discovery + JWKS caching with deterministic refresh windows. +- Password and client-credential flows with token cache abstractions. +- Configurable HTTP retry/backoff policies (Polly) and offline fallback support for air-gapped deployments. + +See `docs/dev/32_AUTH_CLIENT_GUIDE.md` in the repository for integration guidance, option descriptions, and rollout checklists. diff --git a/src/StellaOps.Authority/StellaOps.Auth.Client/ServiceCollectionExtensions.cs b/src/StellaOps.Authority/StellaOps.Auth.Client/ServiceCollectionExtensions.cs new file mode 100644 index 00000000..b3b43c8e --- /dev/null +++ b/src/StellaOps.Authority/StellaOps.Auth.Client/ServiceCollectionExtensions.cs @@ -0,0 +1,115 @@ +using System; +using System.Net; +using System.Net.Http; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.DependencyInjection.Extensions; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Options; +using Polly; +using Polly.Extensions.Http; + +namespace StellaOps.Auth.Client; + +/// +/// DI helpers for the StellaOps auth client. +/// +public static class ServiceCollectionExtensions +{ + /// + /// Registers the StellaOps auth client with the provided configuration. + /// + public static IServiceCollection AddStellaOpsAuthClient(this IServiceCollection services, Action configure) + { + ArgumentNullException.ThrowIfNull(services); + ArgumentNullException.ThrowIfNull(configure); + + services.AddOptions() + .Configure(configure) + .PostConfigure(static options => options.Validate()); + + services.TryAddSingleton(); + + services.AddHttpClient((provider, client) => + { + var options = provider.GetRequiredService>().CurrentValue; + client.Timeout = options.HttpTimeout; + }).AddPolicyHandler(static (provider, _) => CreateRetryPolicy(provider)); + + services.AddHttpClient((provider, client) => + { + var options = provider.GetRequiredService>().CurrentValue; + client.Timeout = options.HttpTimeout; + }).AddPolicyHandler(static (provider, _) => CreateRetryPolicy(provider)); + + services.AddHttpClient((provider, client) => + { + var options = provider.GetRequiredService>().CurrentValue; + client.Timeout = options.HttpTimeout; + }).AddPolicyHandler(static (provider, _) => CreateRetryPolicy(provider)); + + return services; + } + + /// + /// Registers a file-backed token cache implementation. + /// + public static IServiceCollection AddStellaOpsFileTokenCache(this IServiceCollection services, string cacheDirectory) + { + ArgumentNullException.ThrowIfNull(services); + ArgumentException.ThrowIfNullOrWhiteSpace(cacheDirectory); + + services.Replace(ServiceDescriptor.Singleton(provider => + { + var logger = provider.GetService>(); + var options = provider.GetRequiredService>().CurrentValue; + return new FileTokenCache(cacheDirectory, TimeProvider.System, options.ExpirationSkew, logger); + })); + + return services; + } + + private static IAsyncPolicy CreateRetryPolicy(IServiceProvider provider) + { + var options = provider.GetRequiredService>().CurrentValue; + var delays = options.NormalizedRetryDelays; + if (delays.Count == 0) + { + return Policy.NoOpAsync(); + } + + var logger = provider.GetService()?.CreateLogger("StellaOps.Auth.Client.HttpRetry"); + + return HttpPolicyExtensions + .HandleTransientHttpError() + .OrResult(static response => response.StatusCode == HttpStatusCode.TooManyRequests) + .WaitAndRetryAsync( + delays.Count, + attempt => delays[attempt - 1], + (outcome, delay, attempt, _) => + { + if (logger is null) + { + return; + } + + if (outcome.Exception is not null) + { + logger.LogWarning( + outcome.Exception, + "Retrying Authority HTTP call ({Attempt}/{TotalAttempts}) after exception; waiting {Delay}.", + attempt, + delays.Count, + delay); + } + else + { + logger.LogWarning( + "Retrying Authority HTTP call ({Attempt}/{TotalAttempts}) due to status {StatusCode}; waiting {Delay}.", + attempt, + delays.Count, + outcome.Result!.StatusCode, + delay); + } + }); + } +} diff --git a/src/StellaOps.Authority/StellaOps.Auth.Client/StellaOps.Auth.Client.csproj b/src/StellaOps.Authority/StellaOps.Auth.Client/StellaOps.Auth.Client.csproj new file mode 100644 index 00000000..90e81d6f --- /dev/null +++ b/src/StellaOps.Authority/StellaOps.Auth.Client/StellaOps.Auth.Client.csproj @@ -0,0 +1,46 @@ + + + net10.0 + preview + enable + enable + true + + + StellaOps.Auth.Client + Typed OAuth/OpenID client for StellaOps Authority with caching, retries, and token helpers. + StellaOps + StellaOps + AGPL-3.0-or-later + https://stella-ops.org + https://git.stella-ops.org/stella-ops.org/git.stella-ops.org + git + true + true + true + snupkg + stellaops;authentication;authority;oauth2;client + true + $(NoWarn);1591 + README.NuGet.md + 1.0.0-preview.1 + + + + + + + + + + + + + + + + + <_Parameter1>StellaOps.Auth.Client.Tests + + + diff --git a/src/StellaOps.Authority/StellaOps.Auth.Client/StellaOpsAuthClientOptions.cs b/src/StellaOps.Authority/StellaOps.Auth.Client/StellaOpsAuthClientOptions.cs new file mode 100644 index 00000000..089cfcb7 --- /dev/null +++ b/src/StellaOps.Authority/StellaOps.Auth.Client/StellaOpsAuthClientOptions.cs @@ -0,0 +1,205 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using StellaOps.Auth.Abstractions; + +namespace StellaOps.Auth.Client; + +/// +/// Options controlling the StellaOps authentication client. +/// +public sealed class StellaOpsAuthClientOptions +{ + private static readonly TimeSpan[] DefaultRetryDelays = + { + TimeSpan.FromSeconds(1), + TimeSpan.FromSeconds(2), + TimeSpan.FromSeconds(5) + }; + private static readonly TimeSpan DefaultOfflineTolerance = TimeSpan.FromMinutes(10); + + private readonly List scopes = new(); + private readonly List retryDelays = new(DefaultRetryDelays); + + /// + /// Authority (issuer) base URL. + /// + public string Authority { get; set; } = string.Empty; + + /// + /// OAuth client identifier (optional for password flow). + /// + public string ClientId { get; set; } = string.Empty; + + /// + /// OAuth client secret (optional for public clients). + /// + public string? ClientSecret { get; set; } + + /// + /// Default scopes requested for flows that do not explicitly override them. + /// + public IList DefaultScopes => scopes; + + /// + /// Retry delays applied by HTTP retry policy (empty uses defaults). + /// + public IList RetryDelays => retryDelays; + + /// + /// Gets or sets a value indicating whether HTTP retry policies are enabled. + /// + public bool EnableRetries { get; set; } = true; + + /// + /// Timeout applied to discovery and token HTTP requests. + /// + public TimeSpan HttpTimeout { get; set; } = TimeSpan.FromSeconds(30); + + /// + /// Lifetime of cached discovery metadata. + /// + public TimeSpan DiscoveryCacheLifetime { get; set; } = TimeSpan.FromMinutes(10); + + /// + /// Lifetime of cached JWKS metadata. + /// + public TimeSpan JwksCacheLifetime { get; set; } = TimeSpan.FromMinutes(30); + + /// + /// Buffer applied when determining cache expiration (default: 30 seconds). + /// + public TimeSpan ExpirationSkew { get; set; } = TimeSpan.FromSeconds(30); + + /// + /// Gets or sets a value indicating whether cached discovery/JWKS responses may be served when the Authority is unreachable. + /// + public bool AllowOfflineCacheFallback { get; set; } = true; + + /// + /// Additional tolerance window during which stale cache entries remain valid if offline fallback is allowed. + /// + public TimeSpan OfflineCacheTolerance { get; set; } = DefaultOfflineTolerance; + + /// + /// Parsed Authority URI (populated after validation). + /// + public Uri AuthorityUri { get; private set; } = null!; + + /// + /// Normalised scope list (populated after validation). + /// + public IReadOnlyList NormalizedScopes { get; private set; } = Array.Empty(); + + /// + /// Normalised retry delays (populated after validation). + /// + public IReadOnlyList NormalizedRetryDelays { get; private set; } = Array.Empty(); + + /// + /// Validates required values and normalises scope entries. + /// + public void Validate() + { + if (string.IsNullOrWhiteSpace(Authority)) + { + throw new InvalidOperationException("Auth client requires an Authority URL."); + } + + if (!Uri.TryCreate(Authority.Trim(), UriKind.Absolute, out var authorityUri)) + { + throw new InvalidOperationException("Auth client Authority must be an absolute URI."); + } + + if (HttpTimeout <= TimeSpan.Zero) + { + throw new InvalidOperationException("Auth client HTTP timeout must be greater than zero."); + } + + if (DiscoveryCacheLifetime <= TimeSpan.Zero) + { + throw new InvalidOperationException("Discovery cache lifetime must be greater than zero."); + } + + if (JwksCacheLifetime <= TimeSpan.Zero) + { + throw new InvalidOperationException("JWKS cache lifetime must be greater than zero."); + } + + if (ExpirationSkew < TimeSpan.Zero || ExpirationSkew > TimeSpan.FromMinutes(5)) + { + throw new InvalidOperationException("Expiration skew must be between 0 seconds and 5 minutes."); + } + + if (OfflineCacheTolerance < TimeSpan.Zero) + { + throw new InvalidOperationException("Offline cache tolerance must be greater than or equal to zero."); + } + + AuthorityUri = authorityUri; + NormalizedScopes = NormalizeScopes(scopes); + NormalizedRetryDelays = EnableRetries ? NormalizeRetryDelays(retryDelays) : Array.Empty(); + } + + private static IReadOnlyList NormalizeScopes(IList values) + { + if (values.Count == 0) + { + return Array.Empty(); + } + + var unique = new HashSet(StringComparer.Ordinal); + + for (var index = values.Count - 1; index >= 0; index--) + { + var entry = values[index]; + + if (string.IsNullOrWhiteSpace(entry)) + { + values.RemoveAt(index); + continue; + } + + var normalized = StellaOpsScopes.Normalize(entry); + if (normalized is null) + { + values.RemoveAt(index); + continue; + } + + if (!unique.Add(normalized)) + { + values.RemoveAt(index); + continue; + } + + values[index] = normalized; + } + + return values.Count == 0 + ? Array.Empty() + : values.OrderBy(static scope => scope, StringComparer.Ordinal).ToArray(); + } + + private static IReadOnlyList NormalizeRetryDelays(IList values) + { + for (var index = values.Count - 1; index >= 0; index--) + { + var delay = values[index]; + if (delay <= TimeSpan.Zero) + { + values.RemoveAt(index); + } + } + + if (values.Count == 0) + { + foreach (var delay in DefaultRetryDelays) + { + values.Add(delay); + } + } + + return values.ToArray(); + } +} diff --git a/src/StellaOps.Authority/StellaOps.Auth.Client/StellaOpsDiscoveryCache.cs b/src/StellaOps.Authority/StellaOps.Auth.Client/StellaOpsDiscoveryCache.cs new file mode 100644 index 00000000..bfbab405 --- /dev/null +++ b/src/StellaOps.Authority/StellaOps.Auth.Client/StellaOpsDiscoveryCache.cs @@ -0,0 +1,142 @@ +using System; +using System.Net.Http; +using System.Text.Json; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Options; + +namespace StellaOps.Auth.Client; + +/// +/// Caches Authority discovery metadata. +/// +public sealed class StellaOpsDiscoveryCache +{ + private readonly HttpClient httpClient; + private readonly IOptionsMonitor optionsMonitor; + private readonly TimeProvider timeProvider; + private readonly ILogger? logger; + private readonly JsonSerializerOptions serializerOptions = new(JsonSerializerDefaults.Web); + + private OpenIdConfiguration? cachedConfiguration; + private DateTimeOffset cacheExpiresAt; + private DateTimeOffset offlineExpiresAt; + + public StellaOpsDiscoveryCache(HttpClient httpClient, IOptionsMonitor optionsMonitor, TimeProvider? timeProvider = null, ILogger? logger = null) + { + this.httpClient = httpClient ?? throw new ArgumentNullException(nameof(httpClient)); + this.optionsMonitor = optionsMonitor ?? throw new ArgumentNullException(nameof(optionsMonitor)); + this.timeProvider = timeProvider ?? TimeProvider.System; + this.logger = logger; + } + + public async Task GetAsync(CancellationToken cancellationToken) + { + var now = timeProvider.GetUtcNow(); + + if (cachedConfiguration is not null && now < cacheExpiresAt) + { + return cachedConfiguration; + } + + var options = optionsMonitor.CurrentValue; + var discoveryUri = new Uri(options.AuthorityUri, ".well-known/openid-configuration"); + + logger?.LogDebug("Fetching StellaOps discovery document from {DiscoveryUri}.", discoveryUri); + + try + { + using var request = new HttpRequestMessage(HttpMethod.Get, discoveryUri); + using var response = await httpClient.SendAsync(request, cancellationToken).ConfigureAwait(false); + response.EnsureSuccessStatusCode(); + + await using var stream = await response.Content.ReadAsStreamAsync(cancellationToken).ConfigureAwait(false); + var document = await JsonSerializer.DeserializeAsync(stream, serializerOptions, cancellationToken).ConfigureAwait(false); + + if (document is null) + { + throw new InvalidOperationException("Authority discovery document is empty."); + } + + if (string.IsNullOrWhiteSpace(document.TokenEndpoint)) + { + throw new InvalidOperationException("Authority discovery document does not expose token_endpoint."); + } + + if (string.IsNullOrWhiteSpace(document.JwksUri)) + { + throw new InvalidOperationException("Authority discovery document does not expose jwks_uri."); + } + + var configuration = new OpenIdConfiguration( + new Uri(document.TokenEndpoint, UriKind.Absolute), + new Uri(document.JwksUri, UriKind.Absolute)); + + cachedConfiguration = configuration; + cacheExpiresAt = now + options.DiscoveryCacheLifetime; + offlineExpiresAt = cacheExpiresAt + options.OfflineCacheTolerance; + + return configuration; + } + catch (Exception exception) when (IsOfflineCandidate(exception, cancellationToken) && TryUseOfflineFallback(options, now, exception)) + { + return cachedConfiguration!; + } + } + + private sealed record DiscoveryDocument( + [property: System.Text.Json.Serialization.JsonPropertyName("token_endpoint")] string? TokenEndpoint, + [property: System.Text.Json.Serialization.JsonPropertyName("jwks_uri")] string? JwksUri); + + private static bool IsOfflineCandidate(Exception exception, CancellationToken cancellationToken) + { + if (exception is HttpRequestException) + { + return true; + } + + if (exception is TaskCanceledException && !cancellationToken.IsCancellationRequested) + { + return true; + } + + if (exception is TimeoutException) + { + return true; + } + + return false; + } + + private bool TryUseOfflineFallback(StellaOpsAuthClientOptions options, DateTimeOffset now, Exception exception) + { + if (!options.AllowOfflineCacheFallback || cachedConfiguration is null) + { + return false; + } + + if (options.OfflineCacheTolerance <= TimeSpan.Zero) + { + return false; + } + + if (offlineExpiresAt == DateTimeOffset.MinValue) + { + return false; + } + + if (now >= offlineExpiresAt) + { + return false; + } + + logger?.LogWarning(exception, "Discovery document fetch failed; reusing cached configuration until {FallbackExpiresAt}.", offlineExpiresAt); + return true; + } +} + +/// +/// Minimal OpenID Connect configuration representation. +/// +public sealed record OpenIdConfiguration(Uri TokenEndpoint, Uri JwksEndpoint); diff --git a/src/StellaOps.Authority/StellaOps.Auth.Client/StellaOpsJwksCache.cs b/src/StellaOps.Authority/StellaOps.Auth.Client/StellaOpsJwksCache.cs new file mode 100644 index 00000000..dda3ed30 --- /dev/null +++ b/src/StellaOps.Authority/StellaOps.Auth.Client/StellaOpsJwksCache.cs @@ -0,0 +1,116 @@ +using System; +using System.Net.Http; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Options; +using Microsoft.IdentityModel.Tokens; + +namespace StellaOps.Auth.Client; + +/// +/// Caches JWKS documents for Authority. +/// +public sealed class StellaOpsJwksCache +{ + private readonly HttpClient httpClient; + private readonly StellaOpsDiscoveryCache discoveryCache; + private readonly IOptionsMonitor optionsMonitor; + private readonly TimeProvider timeProvider; + private readonly ILogger? logger; + + private JsonWebKeySet? cachedSet; + private DateTimeOffset cacheExpiresAt; + private DateTimeOffset offlineExpiresAt; + + public StellaOpsJwksCache( + HttpClient httpClient, + StellaOpsDiscoveryCache discoveryCache, + IOptionsMonitor optionsMonitor, + TimeProvider? timeProvider = null, + ILogger? logger = null) + { + this.httpClient = httpClient ?? throw new ArgumentNullException(nameof(httpClient)); + this.discoveryCache = discoveryCache ?? throw new ArgumentNullException(nameof(discoveryCache)); + this.optionsMonitor = optionsMonitor ?? throw new ArgumentNullException(nameof(optionsMonitor)); + this.timeProvider = timeProvider ?? TimeProvider.System; + this.logger = logger; + } + + public async Task GetAsync(CancellationToken cancellationToken) + { + var now = timeProvider.GetUtcNow(); + if (cachedSet is not null && now < cacheExpiresAt) + { + return cachedSet; + } + + var options = optionsMonitor.CurrentValue; + var configuration = await discoveryCache.GetAsync(cancellationToken).ConfigureAwait(false); + + logger?.LogDebug("Fetching JWKS from {JwksUri}.", configuration.JwksEndpoint); + + try + { + using var response = await httpClient.GetAsync(configuration.JwksEndpoint, cancellationToken).ConfigureAwait(false); + response.EnsureSuccessStatusCode(); + + var json = await response.Content.ReadAsStringAsync(cancellationToken).ConfigureAwait(false); + cachedSet = new JsonWebKeySet(json); + cacheExpiresAt = now + options.JwksCacheLifetime; + offlineExpiresAt = cacheExpiresAt + options.OfflineCacheTolerance; + + return cachedSet; + } + catch (Exception exception) when (IsOfflineCandidate(exception, cancellationToken) && TryUseOfflineFallback(options, now, exception)) + { + return cachedSet!; + } + } + + private static bool IsOfflineCandidate(Exception exception, CancellationToken cancellationToken) + { + if (exception is HttpRequestException) + { + return true; + } + + if (exception is TaskCanceledException && !cancellationToken.IsCancellationRequested) + { + return true; + } + + if (exception is TimeoutException) + { + return true; + } + + return false; + } + + private bool TryUseOfflineFallback(StellaOpsAuthClientOptions options, DateTimeOffset now, Exception exception) + { + if (!options.AllowOfflineCacheFallback || cachedSet is null) + { + return false; + } + + if (options.OfflineCacheTolerance <= TimeSpan.Zero) + { + return false; + } + + if (offlineExpiresAt == DateTimeOffset.MinValue) + { + return false; + } + + if (now >= offlineExpiresAt) + { + return false; + } + + logger?.LogWarning(exception, "JWKS fetch failed; reusing cached keys until {FallbackExpiresAt}.", offlineExpiresAt); + return true; + } +} diff --git a/src/StellaOps.Authority/StellaOps.Auth.Client/StellaOpsTokenCacheEntry.cs b/src/StellaOps.Authority/StellaOps.Auth.Client/StellaOpsTokenCacheEntry.cs new file mode 100644 index 00000000..4af3f576 --- /dev/null +++ b/src/StellaOps.Authority/StellaOps.Auth.Client/StellaOpsTokenCacheEntry.cs @@ -0,0 +1,49 @@ +using System; +using System.Collections.Generic; +using System.Linq; + +namespace StellaOps.Auth.Client; + +/// +/// Represents a cached token entry. +/// +public sealed record StellaOpsTokenCacheEntry( + string AccessToken, + string TokenType, + DateTimeOffset ExpiresAtUtc, + IReadOnlyList Scopes, + string? RefreshToken = null, + string? IdToken = null, + IReadOnlyDictionary? Metadata = null) +{ + /// + /// Determines whether the token is expired given the provided . + /// + public bool IsExpired(TimeProvider timeProvider, TimeSpan? skew = null) + { + ArgumentNullException.ThrowIfNull(timeProvider); + var now = timeProvider.GetUtcNow(); + var buffer = skew ?? TimeSpan.Zero; + return now >= ExpiresAtUtc - buffer; + } + + /// + /// Creates a copy with scopes normalised. + /// + public StellaOpsTokenCacheEntry NormalizeScopes() + { + if (Scopes.Count == 0) + { + return this; + } + + var normalized = Scopes + .Where(scope => !string.IsNullOrWhiteSpace(scope)) + .Select(scope => scope.Trim()) + .Distinct(StringComparer.Ordinal) + .OrderBy(scope => scope, StringComparer.Ordinal) + .ToArray(); + + return this with { Scopes = normalized }; + } +} diff --git a/src/StellaOps.Authority/StellaOps.Auth.Client/StellaOpsTokenClient.cs b/src/StellaOps.Authority/StellaOps.Auth.Client/StellaOpsTokenClient.cs new file mode 100644 index 00000000..269ebb38 --- /dev/null +++ b/src/StellaOps.Authority/StellaOps.Auth.Client/StellaOpsTokenClient.cs @@ -0,0 +1,205 @@ +using System; +using System.Collections.Generic; +using System.Globalization; +using System.Net.Http; +using System.Net.Http.Headers; +using System.Text; +using System.Text.Json; +using System.Text.Json.Serialization; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Options; +using Microsoft.IdentityModel.Tokens; + +namespace StellaOps.Auth.Client; + +/// +/// Default implementation of . +/// +public sealed class StellaOpsTokenClient : IStellaOpsTokenClient +{ + private static readonly MediaTypeHeaderValue JsonMediaType = new("application/json"); + + private readonly HttpClient httpClient; + private readonly StellaOpsDiscoveryCache discoveryCache; + private readonly StellaOpsJwksCache jwksCache; + private readonly IOptionsMonitor optionsMonitor; + private readonly IStellaOpsTokenCache tokenCache; + private readonly TimeProvider timeProvider; + private readonly ILogger? logger; + private readonly JsonSerializerOptions serializerOptions = new(JsonSerializerDefaults.Web); + + public StellaOpsTokenClient( + HttpClient httpClient, + StellaOpsDiscoveryCache discoveryCache, + StellaOpsJwksCache jwksCache, + IOptionsMonitor optionsMonitor, + IStellaOpsTokenCache tokenCache, + TimeProvider? timeProvider = null, + ILogger? logger = null) + { + this.httpClient = httpClient ?? throw new ArgumentNullException(nameof(httpClient)); + this.discoveryCache = discoveryCache ?? throw new ArgumentNullException(nameof(discoveryCache)); + this.jwksCache = jwksCache ?? throw new ArgumentNullException(nameof(jwksCache)); + this.optionsMonitor = optionsMonitor ?? throw new ArgumentNullException(nameof(optionsMonitor)); + this.tokenCache = tokenCache ?? throw new ArgumentNullException(nameof(tokenCache)); + this.timeProvider = timeProvider ?? TimeProvider.System; + this.logger = logger; + } + + public Task RequestPasswordTokenAsync(string username, string password, string? scope = null, CancellationToken cancellationToken = default) + { + ArgumentException.ThrowIfNullOrWhiteSpace(username); + ArgumentException.ThrowIfNullOrWhiteSpace(password); + + var options = optionsMonitor.CurrentValue; + + var parameters = new Dictionary(StringComparer.Ordinal) + { + ["grant_type"] = "password", + ["username"] = username, + ["password"] = password, + ["client_id"] = options.ClientId + }; + + if (!string.IsNullOrEmpty(options.ClientSecret)) + { + parameters["client_secret"] = options.ClientSecret; + } + + AppendScope(parameters, scope, options); + + return RequestTokenAsync(parameters, cancellationToken); + } + + public Task RequestClientCredentialsTokenAsync(string? scope = null, CancellationToken cancellationToken = default) + { + var options = optionsMonitor.CurrentValue; + if (string.IsNullOrWhiteSpace(options.ClientId)) + { + throw new InvalidOperationException("Client credentials flow requires ClientId to be configured."); + } + + var parameters = new Dictionary(StringComparer.Ordinal) + { + ["grant_type"] = "client_credentials", + ["client_id"] = options.ClientId + }; + + if (!string.IsNullOrEmpty(options.ClientSecret)) + { + parameters["client_secret"] = options.ClientSecret; + } + + AppendScope(parameters, scope, options); + + return RequestTokenAsync(parameters, cancellationToken); + } + + public Task GetJsonWebKeySetAsync(CancellationToken cancellationToken = default) + => jwksCache.GetAsync(cancellationToken); + + public ValueTask GetCachedTokenAsync(string key, CancellationToken cancellationToken = default) + => tokenCache.GetAsync(key, cancellationToken); + + public ValueTask CacheTokenAsync(string key, StellaOpsTokenCacheEntry entry, CancellationToken cancellationToken = default) + => tokenCache.SetAsync(key, entry, cancellationToken); + + public ValueTask ClearCachedTokenAsync(string key, CancellationToken cancellationToken = default) + => tokenCache.RemoveAsync(key, cancellationToken); + + private async Task RequestTokenAsync(Dictionary parameters, CancellationToken cancellationToken) + { + var options = optionsMonitor.CurrentValue; + var configuration = await discoveryCache.GetAsync(cancellationToken).ConfigureAwait(false); + + using var request = new HttpRequestMessage(HttpMethod.Post, configuration.TokenEndpoint) + { + Content = new FormUrlEncodedContent(parameters) + }; + request.Headers.Accept.TryParseAdd(JsonMediaType.ToString()); + + using var response = await httpClient.SendAsync(request, cancellationToken).ConfigureAwait(false); + + var payload = await response.Content.ReadAsStringAsync(cancellationToken).ConfigureAwait(false); + + if (!response.IsSuccessStatusCode) + { + logger?.LogWarning("Token request failed with status {StatusCode}: {Payload}", response.StatusCode, payload); + throw new InvalidOperationException($"Token request failed with status {(int)response.StatusCode}."); + } + + var document = JsonSerializer.Deserialize(payload, serializerOptions); + if (document is null || string.IsNullOrWhiteSpace(document.AccessToken)) + { + throw new InvalidOperationException("Token response did not contain an access_token."); + } + + var expiresIn = document.ExpiresIn ?? 3600; + var expiresAt = timeProvider.GetUtcNow() + TimeSpan.FromSeconds(expiresIn); + var normalizedScopes = ParseScopes(document.Scope ?? parameters.GetValueOrDefault("scope")); + + var result = new StellaOpsTokenResult( + document.AccessToken, + document.TokenType ?? "Bearer", + expiresAt, + normalizedScopes, + document.RefreshToken, + document.IdToken, + payload); + + logger?.LogDebug("Token issued; expires at {ExpiresAt}.", expiresAt); + + return result; + } + + private static void AppendScope(IDictionary parameters, string? scope, StellaOpsAuthClientOptions options) + { + var resolvedScope = scope; + if (string.IsNullOrWhiteSpace(resolvedScope) && options.NormalizedScopes.Count > 0) + { + resolvedScope = string.Join(' ', options.NormalizedScopes); + } + + if (!string.IsNullOrWhiteSpace(resolvedScope)) + { + parameters["scope"] = resolvedScope; + } + } + + private static string[] ParseScopes(string? scope) + { + if (string.IsNullOrWhiteSpace(scope)) + { + return Array.Empty(); + } + + var parts = scope.Split(' ', StringSplitOptions.RemoveEmptyEntries | StringSplitOptions.TrimEntries); + if (parts.Length == 0) + { + return Array.Empty(); + } + + var unique = new HashSet(parts.Length, StringComparer.Ordinal); + foreach (var part in parts) + { + unique.Add(part); + } + + var result = new string[unique.Count]; + unique.CopyTo(result); + Array.Sort(result, StringComparer.Ordinal); + return result; + } + + private sealed record TokenResponseDocument( + [property: JsonPropertyName("access_token")] string? AccessToken, + [property: JsonPropertyName("refresh_token")] string? RefreshToken, + [property: JsonPropertyName("id_token")] string? IdToken, + [property: JsonPropertyName("token_type")] string? TokenType, + [property: JsonPropertyName("expires_in")] int? ExpiresIn, + [property: JsonPropertyName("scope")] string? Scope, + [property: JsonPropertyName("error")] string? Error, + [property: JsonPropertyName("error_description")] string? ErrorDescription); +} diff --git a/src/StellaOps.Authority/StellaOps.Auth.Client/StellaOpsTokenResult.cs b/src/StellaOps.Authority/StellaOps.Auth.Client/StellaOpsTokenResult.cs new file mode 100644 index 00000000..f820adc2 --- /dev/null +++ b/src/StellaOps.Authority/StellaOps.Auth.Client/StellaOpsTokenResult.cs @@ -0,0 +1,23 @@ +using System; +using System.Collections.Generic; + +namespace StellaOps.Auth.Client; + +/// +/// Represents an issued token with metadata. +/// +public sealed record StellaOpsTokenResult( + string AccessToken, + string TokenType, + DateTimeOffset ExpiresAtUtc, + IReadOnlyList Scopes, + string? RefreshToken = null, + string? IdToken = null, + string? RawResponse = null) +{ + /// + /// Converts the result to a cache entry. + /// + public StellaOpsTokenCacheEntry ToCacheEntry() + => new(AccessToken, TokenType, ExpiresAtUtc, Scopes, RefreshToken, IdToken); +} diff --git a/src/StellaOps.Authority/StellaOps.Auth.ServerIntegration.Tests/ServiceCollectionExtensionsTests.cs b/src/StellaOps.Authority/StellaOps.Auth.ServerIntegration.Tests/ServiceCollectionExtensionsTests.cs new file mode 100644 index 00000000..c0477f2a --- /dev/null +++ b/src/StellaOps.Authority/StellaOps.Auth.ServerIntegration.Tests/ServiceCollectionExtensionsTests.cs @@ -0,0 +1,44 @@ +using System; +using System.Collections.Generic; +using Microsoft.AspNetCore.Authentication.JwtBearer; +using Microsoft.Extensions.Configuration; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Options; +using StellaOps.Auth.Abstractions; +using StellaOps.Auth.ServerIntegration; +using Xunit; + +namespace StellaOps.Auth.ServerIntegration.Tests; + +public class ServiceCollectionExtensionsTests +{ + [Fact] + public void AddStellaOpsResourceServerAuthentication_ConfiguresJwtBearer() + { + var configuration = new ConfigurationBuilder() + .AddInMemoryCollection(new Dictionary + { + ["Authority:ResourceServer:Authority"] = "https://authority.example", + ["Authority:ResourceServer:Audiences:0"] = "api://feedser", + ["Authority:ResourceServer:RequiredScopes:0"] = "feedser.jobs.trigger", + ["Authority:ResourceServer:BypassNetworks:0"] = "127.0.0.1/32" + }) + .Build(); + + var services = new ServiceCollection(); + services.AddLogging(); + services.AddStellaOpsResourceServerAuthentication(configuration); + + using var provider = services.BuildServiceProvider(); + + var resourceOptions = provider.GetRequiredService>().CurrentValue; + var jwtOptions = provider.GetRequiredService>().Get(StellaOpsAuthenticationDefaults.AuthenticationScheme); + + Assert.NotNull(jwtOptions.Authority); + Assert.Equal(new Uri("https://authority.example/"), new Uri(jwtOptions.Authority!)); + Assert.True(jwtOptions.TokenValidationParameters.ValidateAudience); + Assert.Contains("api://feedser", jwtOptions.TokenValidationParameters.ValidAudiences); + Assert.Equal(TimeSpan.FromSeconds(60), jwtOptions.TokenValidationParameters.ClockSkew); + Assert.Equal(new[] { "feedser.jobs.trigger" }, resourceOptions.NormalizedScopes); + } +} diff --git a/src/StellaOps.Authority/StellaOps.Auth.ServerIntegration.Tests/StellaOps.Auth.ServerIntegration.Tests.csproj b/src/StellaOps.Authority/StellaOps.Auth.ServerIntegration.Tests/StellaOps.Auth.ServerIntegration.Tests.csproj new file mode 100644 index 00000000..9109d167 --- /dev/null +++ b/src/StellaOps.Authority/StellaOps.Auth.ServerIntegration.Tests/StellaOps.Auth.ServerIntegration.Tests.csproj @@ -0,0 +1,11 @@ + + + net10.0 + enable + enable + + + + + + diff --git a/src/StellaOps.Authority/StellaOps.Auth.ServerIntegration.Tests/StellaOpsResourceServerOptionsTests.cs b/src/StellaOps.Authority/StellaOps.Auth.ServerIntegration.Tests/StellaOpsResourceServerOptionsTests.cs new file mode 100644 index 00000000..91454125 --- /dev/null +++ b/src/StellaOps.Authority/StellaOps.Auth.ServerIntegration.Tests/StellaOpsResourceServerOptionsTests.cs @@ -0,0 +1,50 @@ +using System; +using System.Net; +using StellaOps.Auth.ServerIntegration; +using Xunit; + +namespace StellaOps.Auth.ServerIntegration.Tests; + +public class StellaOpsResourceServerOptionsTests +{ + [Fact] + public void Validate_NormalisesCollections() + { + var options = new StellaOpsResourceServerOptions + { + Authority = "https://authority.stella-ops.test", + BackchannelTimeout = TimeSpan.FromSeconds(10), + TokenClockSkew = TimeSpan.FromSeconds(30) + }; + + options.Audiences.Add(" api://feedser "); + options.Audiences.Add("api://feedser"); + options.Audiences.Add("api://feedser-admin"); + + options.RequiredScopes.Add(" Feedser.Jobs.Trigger "); + options.RequiredScopes.Add("feedser.jobs.trigger"); + options.RequiredScopes.Add("AUTHORITY.USERS.MANAGE"); + + options.BypassNetworks.Add("127.0.0.1/32"); + options.BypassNetworks.Add(" 127.0.0.1/32 "); + options.BypassNetworks.Add("::1/128"); + + options.Validate(); + + Assert.Equal(new Uri("https://authority.stella-ops.test"), options.AuthorityUri); + Assert.Equal(new[] { "api://feedser", "api://feedser-admin" }, options.Audiences); + Assert.Equal(new[] { "authority.users.manage", "feedser.jobs.trigger" }, options.NormalizedScopes); + Assert.True(options.BypassMatcher.IsAllowed(IPAddress.Parse("127.0.0.1"))); + Assert.True(options.BypassMatcher.IsAllowed(IPAddress.IPv6Loopback)); + } + + [Fact] + public void Validate_Throws_When_AuthorityMissing() + { + var options = new StellaOpsResourceServerOptions(); + + var exception = Assert.Throws(() => options.Validate()); + + Assert.Contains("Authority", exception.Message, StringComparison.OrdinalIgnoreCase); + } +} diff --git a/src/StellaOps.Authority/StellaOps.Auth.ServerIntegration.Tests/StellaOpsScopeAuthorizationHandlerTests.cs b/src/StellaOps.Authority/StellaOps.Auth.ServerIntegration.Tests/StellaOpsScopeAuthorizationHandlerTests.cs new file mode 100644 index 00000000..534a6ef0 --- /dev/null +++ b/src/StellaOps.Authority/StellaOps.Auth.ServerIntegration.Tests/StellaOpsScopeAuthorizationHandlerTests.cs @@ -0,0 +1,123 @@ +using System; +using System.Net; +using System.Security.Claims; +using System.Threading.Tasks; +using Microsoft.AspNetCore.Authorization; +using Microsoft.AspNetCore.Http; +using Microsoft.Extensions.Logging.Abstractions; +using Microsoft.Extensions.Options; +using StellaOps.Auth.Abstractions; +using StellaOps.Auth.ServerIntegration; +using Xunit; + +namespace StellaOps.Auth.ServerIntegration.Tests; + +public class StellaOpsScopeAuthorizationHandlerTests +{ + [Fact] + public async Task HandleRequirement_Succeeds_WhenScopePresent() + { + var optionsMonitor = CreateOptionsMonitor(options => + { + options.Authority = "https://authority.example"; + options.Validate(); + }); + + var (handler, accessor) = CreateHandler(optionsMonitor, remoteAddress: IPAddress.Parse("10.0.0.1")); + var requirement = new StellaOpsScopeRequirement(new[] { StellaOpsScopes.FeedserJobsTrigger }); + var principal = new StellaOpsPrincipalBuilder() + .WithSubject("user-1") + .WithScopes(new[] { StellaOpsScopes.FeedserJobsTrigger }) + .Build(); + + var context = new AuthorizationHandlerContext(new[] { requirement }, principal, accessor.HttpContext); + + await handler.HandleAsync(context); + + Assert.True(context.HasSucceeded); + } + + [Fact] + public async Task HandleRequirement_Succeeds_WhenBypassNetworkMatches() + { + var optionsMonitor = CreateOptionsMonitor(options => + { + options.Authority = "https://authority.example"; + options.BypassNetworks.Add("127.0.0.1/32"); + options.Validate(); + }); + + var (handler, accessor) = CreateHandler(optionsMonitor, remoteAddress: IPAddress.Parse("127.0.0.1")); + var requirement = new StellaOpsScopeRequirement(new[] { StellaOpsScopes.FeedserJobsTrigger }); + var principal = new ClaimsPrincipal(new ClaimsIdentity()); + var context = new AuthorizationHandlerContext(new[] { requirement }, principal, accessor.HttpContext); + + await handler.HandleAsync(context); + + Assert.True(context.HasSucceeded); + } + + [Fact] + public async Task HandleRequirement_Fails_WhenScopeMissingAndNoBypass() + { + var optionsMonitor = CreateOptionsMonitor(options => + { + options.Authority = "https://authority.example"; + options.Validate(); + }); + + var (handler, accessor) = CreateHandler(optionsMonitor, remoteAddress: IPAddress.Parse("203.0.113.10")); + var requirement = new StellaOpsScopeRequirement(new[] { StellaOpsScopes.FeedserJobsTrigger }); + var principal = new ClaimsPrincipal(new ClaimsIdentity()); + var context = new AuthorizationHandlerContext(new[] { requirement }, principal, accessor.HttpContext); + + await handler.HandleAsync(context); + + Assert.False(context.HasSucceeded); + } + + private static (StellaOpsScopeAuthorizationHandler Handler, IHttpContextAccessor Accessor) CreateHandler(IOptionsMonitor optionsMonitor, IPAddress remoteAddress) + { + var accessor = new HttpContextAccessor(); + var httpContext = new DefaultHttpContext(); + httpContext.Connection.RemoteIpAddress = remoteAddress; + accessor.HttpContext = httpContext; + + var bypassEvaluator = new StellaOpsBypassEvaluator(optionsMonitor, NullLogger.Instance); + + var handler = new StellaOpsScopeAuthorizationHandler( + accessor, + bypassEvaluator, + NullLogger.Instance); + return (handler, accessor); + } + + private static IOptionsMonitor CreateOptionsMonitor(Action configure) + => new TestOptionsMonitor(configure); + + private sealed class TestOptionsMonitor : IOptionsMonitor + where TOptions : class, new() + { + private readonly TOptions value; + + public TestOptionsMonitor(Action configure) + { + value = new TOptions(); + configure(value); + } + + public TOptions CurrentValue => value; + + public TOptions Get(string? name) => value; + + public IDisposable OnChange(Action listener) => NullDisposable.Instance; + + private sealed class NullDisposable : IDisposable + { + public static NullDisposable Instance { get; } = new(); + public void Dispose() + { + } + } + } +} diff --git a/src/StellaOps.Authority/StellaOps.Auth.ServerIntegration/README.NuGet.md b/src/StellaOps.Authority/StellaOps.Auth.ServerIntegration/README.NuGet.md new file mode 100644 index 00000000..1f7eba7f --- /dev/null +++ b/src/StellaOps.Authority/StellaOps.Auth.ServerIntegration/README.NuGet.md @@ -0,0 +1,9 @@ +# StellaOps.Auth.ServerIntegration + +ASP.NET Core helpers that enable resource servers to authenticate with **StellaOps Authority**: + +- `AddStellaOpsResourceServerAuthentication` extension for JWT bearer + scope policies. +- Network bypass mask evaluation for on-host automation. +- Consistent `ProblemDetails` responses and policy helpers shared with Feedser/Backend services. + +Pair this package with `StellaOps.Auth.Abstractions` and `StellaOps.Auth.Client` for end-to-end Authority integration. diff --git a/src/StellaOps.Authority/StellaOps.Auth.ServerIntegration/ServiceCollectionExtensions.cs b/src/StellaOps.Authority/StellaOps.Auth.ServerIntegration/ServiceCollectionExtensions.cs new file mode 100644 index 00000000..5c620356 --- /dev/null +++ b/src/StellaOps.Authority/StellaOps.Auth.ServerIntegration/ServiceCollectionExtensions.cs @@ -0,0 +1,88 @@ +using System; +using System.Security.Claims; +using Microsoft.AspNetCore.Authentication.JwtBearer; +using Microsoft.Extensions.Configuration; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.DependencyInjection.Extensions; +using Microsoft.Extensions.Options; +using Microsoft.IdentityModel.Tokens; +using StellaOps.Auth.Abstractions; + +namespace StellaOps.Auth.ServerIntegration; + +/// +/// Dependency injection helpers for configuring StellaOps resource server authentication. +/// +public static class ServiceCollectionExtensions +{ + /// + /// Registers JWT bearer authentication and related authorisation helpers using the provided configuration section. + /// + /// The service collection. + /// Application configuration. + /// + /// Optional configuration section path. Defaults to Authority:ResourceServer. Provide null to skip binding. + /// + /// Optional callback allowing additional mutation of . + public static IServiceCollection AddStellaOpsResourceServerAuthentication( + this IServiceCollection services, + IConfiguration configuration, + string? configurationSection = "Authority:ResourceServer", + Action? configure = null) + { + ArgumentNullException.ThrowIfNull(services); + ArgumentNullException.ThrowIfNull(configuration); + + services.AddHttpContextAccessor(); + services.AddAuthorization(); + services.AddStellaOpsScopeHandler(); + services.TryAddSingleton(); + + var optionsBuilder = services.AddOptions(); + if (!string.IsNullOrWhiteSpace(configurationSection)) + { + optionsBuilder.Bind(configuration.GetSection(configurationSection)); + } + + if (configure is not null) + { + optionsBuilder.Configure(configure); + } + + optionsBuilder.PostConfigure(static options => options.Validate()); + + var authenticationBuilder = services.AddAuthentication(options => + { + options.DefaultAuthenticateScheme ??= StellaOpsAuthenticationDefaults.AuthenticationScheme; + options.DefaultChallengeScheme ??= StellaOpsAuthenticationDefaults.AuthenticationScheme; + }); + + authenticationBuilder.AddJwtBearer(StellaOpsAuthenticationDefaults.AuthenticationScheme); + + services.AddOptions(StellaOpsAuthenticationDefaults.AuthenticationScheme) + .Configure>((jwt, monitor) => + { + var resourceOptions = monitor.CurrentValue; + + jwt.Authority = resourceOptions.AuthorityUri.ToString(); + if (!string.IsNullOrWhiteSpace(resourceOptions.MetadataAddress)) + { + jwt.MetadataAddress = resourceOptions.MetadataAddress; + } + jwt.RequireHttpsMetadata = resourceOptions.RequireHttpsMetadata; + jwt.BackchannelTimeout = resourceOptions.BackchannelTimeout; + jwt.MapInboundClaims = false; + jwt.SaveToken = false; + + jwt.TokenValidationParameters ??= new TokenValidationParameters(); + jwt.TokenValidationParameters.ValidIssuer = resourceOptions.AuthorityUri.ToString(); + jwt.TokenValidationParameters.ValidateAudience = resourceOptions.Audiences.Count > 0; + jwt.TokenValidationParameters.ValidAudiences = resourceOptions.Audiences; + jwt.TokenValidationParameters.ClockSkew = resourceOptions.TokenClockSkew; + jwt.TokenValidationParameters.NameClaimType = ClaimTypes.Name; + jwt.TokenValidationParameters.RoleClaimType = ClaimTypes.Role; + }); + + return services; + } +} diff --git a/src/StellaOps.Authority/StellaOps.Auth.ServerIntegration/StellaOps.Auth.ServerIntegration.csproj b/src/StellaOps.Authority/StellaOps.Auth.ServerIntegration/StellaOps.Auth.ServerIntegration.csproj new file mode 100644 index 00000000..33cf5438 --- /dev/null +++ b/src/StellaOps.Authority/StellaOps.Auth.ServerIntegration/StellaOps.Auth.ServerIntegration.csproj @@ -0,0 +1,46 @@ + + + net10.0 + preview + enable + enable + true + + + StellaOps.Auth.ServerIntegration + ASP.NET server integration helpers for StellaOps Authority, including JWT validation and bypass masks. + StellaOps + StellaOps + AGPL-3.0-or-later + https://stella-ops.org + https://git.stella-ops.org/stella-ops.org/git.stella-ops.org + git + true + true + true + snupkg + stellaops;authentication;authority;aspnet + true + $(NoWarn);1591 + README.NuGet.md + 1.0.0-preview.1 + + + + + + + + + + + + + + + + + <_Parameter1>StellaOps.Auth.ServerIntegration.Tests + + + diff --git a/src/StellaOps.Authority/StellaOps.Auth.ServerIntegration/StellaOpsAuthorizationPolicyBuilderExtensions.cs b/src/StellaOps.Authority/StellaOps.Auth.ServerIntegration/StellaOpsAuthorizationPolicyBuilderExtensions.cs new file mode 100644 index 00000000..319562d6 --- /dev/null +++ b/src/StellaOps.Authority/StellaOps.Auth.ServerIntegration/StellaOpsAuthorizationPolicyBuilderExtensions.cs @@ -0,0 +1,56 @@ +using System; +using Microsoft.AspNetCore.Authorization; +using Microsoft.Extensions.DependencyInjection; +using StellaOps.Auth.Abstractions; + +namespace StellaOps.Auth.ServerIntegration; + +/// +/// Extension methods for configuring StellaOps authorisation policies. +/// +public static class StellaOpsAuthorizationPolicyBuilderExtensions +{ + /// + /// Requires the specified scopes using the StellaOps scope requirement. + /// + public static AuthorizationPolicyBuilder RequireStellaOpsScopes( + this AuthorizationPolicyBuilder builder, + params string[] scopes) + { + ArgumentNullException.ThrowIfNull(builder); + + var requirement = new StellaOpsScopeRequirement(scopes); + builder.AddRequirements(requirement); + builder.AddAuthenticationSchemes(StellaOpsAuthenticationDefaults.AuthenticationScheme); + return builder; + } + + /// + /// Registers a named policy that enforces the provided scopes. + /// + public static void AddStellaOpsScopePolicy( + this AuthorizationOptions options, + string policyName, + params string[] scopes) + { + ArgumentNullException.ThrowIfNull(options); + ArgumentException.ThrowIfNullOrWhiteSpace(policyName); + + options.AddPolicy(policyName, policy => + { + policy.AddAuthenticationSchemes(StellaOpsAuthenticationDefaults.AuthenticationScheme); + policy.Requirements.Add(new StellaOpsScopeRequirement(scopes)); + }); + } + + /// + /// Adds the scope handler to the DI container. + /// + public static IServiceCollection AddStellaOpsScopeHandler(this IServiceCollection services) + { + ArgumentNullException.ThrowIfNull(services); + + services.AddSingleton(); + return services; + } +} diff --git a/src/StellaOps.Authority/StellaOps.Auth.ServerIntegration/StellaOpsBypassEvaluator.cs b/src/StellaOps.Authority/StellaOps.Auth.ServerIntegration/StellaOpsBypassEvaluator.cs new file mode 100644 index 00000000..826d81af --- /dev/null +++ b/src/StellaOps.Authority/StellaOps.Auth.ServerIntegration/StellaOpsBypassEvaluator.cs @@ -0,0 +1,62 @@ +using System; +using System.Collections.Generic; +using Microsoft.AspNetCore.Http; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Options; + +namespace StellaOps.Auth.ServerIntegration; + +/// +/// Evaluates whether a request qualifies for network-based bypass. +/// +public sealed class StellaOpsBypassEvaluator +{ + private readonly IOptionsMonitor optionsMonitor; + private readonly ILogger logger; + + public StellaOpsBypassEvaluator( + IOptionsMonitor optionsMonitor, + ILogger logger) + { + this.optionsMonitor = optionsMonitor; + this.logger = logger; + } + + public bool ShouldBypass(HttpContext context, IReadOnlyCollection requiredScopes) + { + ArgumentNullException.ThrowIfNull(context); + + var options = optionsMonitor.CurrentValue; + var matcher = options.BypassMatcher; + + if (matcher.IsEmpty) + { + return false; + } + + var remoteAddress = context.Connection.RemoteIpAddress; + if (remoteAddress is null) + { + logger.LogDebug("Bypass skipped because remote IP address is unavailable."); + return false; + } + + if (!matcher.IsAllowed(remoteAddress)) + { + return false; + } + + if (context.Request.Headers.ContainsKey("Authorization")) + { + logger.LogDebug("Bypass skipped because Authorization header is present for {RemoteIp}.", remoteAddress); + return false; + } + + logger.LogInformation( + "Granting StellaOps bypass for remote {RemoteIp}; required scopes {RequiredScopes}.", + remoteAddress, + string.Join(", ", requiredScopes)); + + return true; + } +} diff --git a/src/StellaOps.Authority/StellaOps.Auth.ServerIntegration/StellaOpsResourceServerOptions.cs b/src/StellaOps.Authority/StellaOps.Auth.ServerIntegration/StellaOpsResourceServerOptions.cs new file mode 100644 index 00000000..f4957457 --- /dev/null +++ b/src/StellaOps.Authority/StellaOps.Auth.ServerIntegration/StellaOpsResourceServerOptions.cs @@ -0,0 +1,152 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using StellaOps.Auth.Abstractions; + +namespace StellaOps.Auth.ServerIntegration; + +/// +/// Options controlling StellaOps resource server authentication. +/// +public sealed class StellaOpsResourceServerOptions +{ + private readonly List audiences = new(); + private readonly List requiredScopes = new(); + private readonly List bypassNetworks = new(); + + /// + /// Gets or sets the Authority (issuer) URL that exposes OpenID discovery. + /// + public string Authority { get; set; } = string.Empty; + + /// + /// Optional explicit OpenID Connect metadata address. + /// + public string? MetadataAddress { get; set; } + + /// + /// Audiences accepted by the resource server (validated against the aud claim). + /// + public IList Audiences => audiences; + + /// + /// Scopes enforced by default authorisation policies. + /// + public IList RequiredScopes => requiredScopes; + + /// + /// Networks permitted to bypass authentication (used for trusted on-host automation). + /// + public IList BypassNetworks => bypassNetworks; + + /// + /// Whether HTTPS metadata is required when communicating with Authority. + /// + public bool RequireHttpsMetadata { get; set; } = true; + + /// + /// Back-channel timeout when fetching metadata/JWKS. + /// + public TimeSpan BackchannelTimeout { get; set; } = TimeSpan.FromSeconds(30); + + /// + /// Clock skew tolerated when validating tokens. + /// + public TimeSpan TokenClockSkew { get; set; } = TimeSpan.FromSeconds(60); + + /// + /// Gets the canonical Authority URI (populated during validation). + /// + public Uri AuthorityUri { get; private set; } = null!; + + /// + /// Gets the normalised scope list (populated during validation). + /// + public IReadOnlyList NormalizedScopes { get; private set; } = Array.Empty(); + + /// + /// Gets the network matcher used for bypass checks (populated during validation). + /// + public NetworkMaskMatcher BypassMatcher { get; private set; } = NetworkMaskMatcher.DenyAll; + + /// + /// Validates provided configuration and normalises collections. + /// + public void Validate() + { + if (string.IsNullOrWhiteSpace(Authority)) + { + throw new InvalidOperationException("Resource server authentication requires an Authority URL."); + } + + if (!Uri.TryCreate(Authority.Trim(), UriKind.Absolute, out var authorityUri)) + { + throw new InvalidOperationException("Resource server Authority URL must be an absolute URI."); + } + + if (RequireHttpsMetadata && + !authorityUri.IsLoopback && + !string.Equals(authorityUri.Scheme, Uri.UriSchemeHttps, StringComparison.OrdinalIgnoreCase)) + { + throw new InvalidOperationException("Resource server Authority URL must use HTTPS when HTTPS metadata is required."); + } + + if (BackchannelTimeout <= TimeSpan.Zero) + { + throw new InvalidOperationException("Resource server back-channel timeout must be greater than zero."); + } + + if (TokenClockSkew < TimeSpan.Zero || TokenClockSkew > TimeSpan.FromMinutes(5)) + { + throw new InvalidOperationException("Resource server token clock skew must be between 0 seconds and 5 minutes."); + } + + AuthorityUri = authorityUri; + + NormalizeList(audiences, toLower: false); + NormalizeList(requiredScopes, toLower: true); + NormalizeList(bypassNetworks, toLower: false); + + NormalizedScopes = requiredScopes.Count == 0 + ? Array.Empty() + : requiredScopes.OrderBy(static scope => scope, StringComparer.Ordinal).ToArray(); + + BypassMatcher = bypassNetworks.Count == 0 + ? NetworkMaskMatcher.DenyAll + : new NetworkMaskMatcher(bypassNetworks); + } + + private static void NormalizeList(IList values, bool toLower) + { + if (values.Count == 0) + { + return; + } + + var seen = new HashSet(StringComparer.OrdinalIgnoreCase); + + for (var index = values.Count - 1; index >= 0; index--) + { + var value = values[index]; + if (string.IsNullOrWhiteSpace(value)) + { + values.RemoveAt(index); + continue; + } + + var trimmed = value.Trim(); + if (toLower) + { + trimmed = trimmed.ToLowerInvariant(); + } + + if (!seen.Add(trimmed)) + { + values.RemoveAt(index); + continue; + } + + values[index] = trimmed; + } + } +} diff --git a/src/StellaOps.Authority/StellaOps.Auth.ServerIntegration/StellaOpsScopeAuthorizationHandler.cs b/src/StellaOps.Authority/StellaOps.Auth.ServerIntegration/StellaOpsScopeAuthorizationHandler.cs new file mode 100644 index 00000000..519950f3 --- /dev/null +++ b/src/StellaOps.Authority/StellaOps.Auth.ServerIntegration/StellaOpsScopeAuthorizationHandler.cs @@ -0,0 +1,111 @@ +using System; +using System.Collections.Generic; +using System.Security.Claims; +using System.Threading.Tasks; +using Microsoft.AspNetCore.Authorization; +using Microsoft.AspNetCore.Http; +using Microsoft.Extensions.Logging; +using StellaOps.Auth.Abstractions; + +namespace StellaOps.Auth.ServerIntegration; + +/// +/// Handles evaluation. +/// +internal sealed class StellaOpsScopeAuthorizationHandler : AuthorizationHandler +{ + private readonly IHttpContextAccessor httpContextAccessor; + private readonly StellaOpsBypassEvaluator bypassEvaluator; + private readonly ILogger logger; + + public StellaOpsScopeAuthorizationHandler( + IHttpContextAccessor httpContextAccessor, + StellaOpsBypassEvaluator bypassEvaluator, + ILogger logger) + { + this.httpContextAccessor = httpContextAccessor; + this.bypassEvaluator = bypassEvaluator; + this.logger = logger; + } + + protected override Task HandleRequirementAsync( + AuthorizationHandlerContext context, + StellaOpsScopeRequirement requirement) + { + HashSet? userScopes = null; + + if (context.User?.Identity?.IsAuthenticated == true) + { + userScopes = ExtractScopes(context.User); + + foreach (var scope in requirement.RequiredScopes) + { + if (userScopes.Contains(scope)) + { + context.Succeed(requirement); + return Task.CompletedTask; + } + } + } + + var httpContext = httpContextAccessor.HttpContext; + + if (httpContext is not null && bypassEvaluator.ShouldBypass(httpContext, requirement.RequiredScopes)) + { + context.Succeed(requirement); + return Task.CompletedTask; + } + + if (logger.IsEnabled(LogLevel.Debug)) + { + var required = string.Join(", ", requirement.RequiredScopes); + var principalScopes = userScopes is null || userScopes.Count == 0 + ? "(none)" + : string.Join(", ", userScopes); + + logger.LogDebug( + "Scope requirement not satisfied. Required={RequiredScopes}; PrincipalScopes={PrincipalScopes}; Remote={Remote}", + required, + principalScopes, + httpContext?.Connection.RemoteIpAddress); + } + + return Task.CompletedTask; + } + + private static HashSet ExtractScopes(ClaimsPrincipal principal) + { + var scopes = new HashSet(StringComparer.Ordinal); + + foreach (var claim in principal.FindAll(StellaOpsClaimTypes.ScopeItem)) + { + if (string.IsNullOrWhiteSpace(claim.Value)) + { + continue; + } + + scopes.Add(claim.Value); + } + + foreach (var claim in principal.FindAll(StellaOpsClaimTypes.Scope)) + { + if (string.IsNullOrWhiteSpace(claim.Value)) + { + continue; + } + + var parts = claim.Value.Split(' ', StringSplitOptions.RemoveEmptyEntries | StringSplitOptions.TrimEntries); + + foreach (var part in parts) + { + var normalized = StellaOpsScopes.Normalize(part); + if (normalized is not null) + { + scopes.Add(normalized); + } + } + } + + return scopes; + } +} diff --git a/src/StellaOps.Authority/StellaOps.Auth.ServerIntegration/StellaOpsScopeRequirement.cs b/src/StellaOps.Authority/StellaOps.Auth.ServerIntegration/StellaOpsScopeRequirement.cs new file mode 100644 index 00000000..69796e78 --- /dev/null +++ b/src/StellaOps.Authority/StellaOps.Auth.ServerIntegration/StellaOpsScopeRequirement.cs @@ -0,0 +1,47 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using Microsoft.AspNetCore.Authorization; +using StellaOps.Auth.Abstractions; + +namespace StellaOps.Auth.ServerIntegration; + +/// +/// Authorisation requirement enforcing StellaOps scope membership. +/// +public sealed class StellaOpsScopeRequirement : IAuthorizationRequirement +{ + /// + /// Initialises a new instance of the class. + /// + /// Scopes that satisfy the requirement. + public StellaOpsScopeRequirement(IEnumerable scopes) + { + ArgumentNullException.ThrowIfNull(scopes); + + var normalized = new HashSet(StringComparer.Ordinal); + + foreach (var scope in scopes) + { + var value = StellaOpsScopes.Normalize(scope); + if (value is null) + { + continue; + } + + normalized.Add(value); + } + + if (normalized.Count == 0) + { + throw new ArgumentException("At least one scope must be provided.", nameof(scopes)); + } + + RequiredScopes = normalized.OrderBy(static scope => scope, StringComparer.Ordinal).ToArray(); + } + + /// + /// Gets the required scopes. + /// + public IReadOnlyCollection RequiredScopes { get; } +} diff --git a/src/StellaOps.Authority/StellaOps.Authority.Plugin.Standard.Tests/StandardClientProvisioningStoreTests.cs b/src/StellaOps.Authority/StellaOps.Authority.Plugin.Standard.Tests/StandardClientProvisioningStoreTests.cs new file mode 100644 index 00000000..125d2c17 --- /dev/null +++ b/src/StellaOps.Authority/StellaOps.Authority.Plugin.Standard.Tests/StandardClientProvisioningStoreTests.cs @@ -0,0 +1,66 @@ +using System.Collections.Generic; +using System.Threading; +using System.Threading.Tasks; +using StellaOps.Authority.Plugins.Abstractions; +using StellaOps.Authority.Plugin.Standard.Storage; +using StellaOps.Authority.Storage.Mongo.Documents; +using StellaOps.Authority.Storage.Mongo.Stores; +using Xunit; + +namespace StellaOps.Authority.Plugin.Standard.Tests; + +public class StandardClientProvisioningStoreTests +{ + [Fact] + public async Task CreateOrUpdateAsync_HashesSecretAndPersistsDocument() + { + var store = new TrackingClientStore(); + var provisioning = new StandardClientProvisioningStore("standard", store); + + var registration = new AuthorityClientRegistration( + clientId: "bootstrap-client", + confidential: true, + displayName: "Bootstrap", + clientSecret: "SuperSecret1!", + allowedGrantTypes: new[] { "client_credentials" }, + allowedScopes: new[] { "scopeA" }); + + var result = await provisioning.CreateOrUpdateAsync(registration, CancellationToken.None); + + Assert.True(result.Succeeded); + Assert.True(store.Documents.TryGetValue("bootstrap-client", out var document)); + Assert.NotNull(document); + Assert.Equal(AuthoritySecretHasher.ComputeHash("SuperSecret1!"), document!.SecretHash); + Assert.Equal("standard", document.Plugin); + + var descriptor = await provisioning.FindByClientIdAsync("bootstrap-client", CancellationToken.None); + Assert.NotNull(descriptor); + Assert.Equal("bootstrap-client", descriptor!.ClientId); + Assert.True(descriptor.Confidential); + Assert.Contains("client_credentials", descriptor.AllowedGrantTypes); + Assert.Contains("scopeA", descriptor.AllowedScopes); + } + + private sealed class TrackingClientStore : IAuthorityClientStore + { + public Dictionary Documents { get; } = new(StringComparer.OrdinalIgnoreCase); + + public ValueTask FindByClientIdAsync(string clientId, CancellationToken cancellationToken) + { + Documents.TryGetValue(clientId, out var document); + return ValueTask.FromResult(document); + } + + public ValueTask UpsertAsync(AuthorityClientDocument document, CancellationToken cancellationToken) + { + Documents[document.ClientId] = document; + return ValueTask.CompletedTask; + } + + public ValueTask DeleteByClientIdAsync(string clientId, CancellationToken cancellationToken) + { + var removed = Documents.Remove(clientId); + return ValueTask.FromResult(removed); + } + } +} diff --git a/src/StellaOps.Authority/StellaOps.Authority.Plugin.Standard.Tests/StandardPluginOptionsTests.cs b/src/StellaOps.Authority/StellaOps.Authority.Plugin.Standard.Tests/StandardPluginOptionsTests.cs new file mode 100644 index 00000000..f0f32058 --- /dev/null +++ b/src/StellaOps.Authority/StellaOps.Authority.Plugin.Standard.Tests/StandardPluginOptionsTests.cs @@ -0,0 +1,99 @@ +using System; +using System.IO; +using StellaOps.Authority.Plugin.Standard; + +namespace StellaOps.Authority.Plugin.Standard.Tests; + +public class StandardPluginOptionsTests +{ + [Fact] + public void Validate_AllowsBootstrapWhenCredentialsProvided() + { + var options = new StandardPluginOptions + { + BootstrapUser = new BootstrapUserOptions + { + Username = "admin", + Password = "Bootstrap1!", + RequirePasswordReset = true + } + }; + + options.Validate("standard"); + } + + [Fact] + public void Validate_Throws_WhenBootstrapUserIncomplete() + { + var options = new StandardPluginOptions + { + BootstrapUser = new BootstrapUserOptions + { + Username = "admin", + Password = null + } + }; + + var ex = Assert.Throws(() => options.Validate("standard")); + Assert.Contains("bootstrapUser", ex.Message, StringComparison.OrdinalIgnoreCase); + } + + [Fact] + public void Validate_Throws_WhenLockoutWindowMinutesInvalid() + { + var options = new StandardPluginOptions + { + Lockout = new LockoutOptions + { + Enabled = true, + MaxAttempts = 5, + WindowMinutes = 0 + } + }; + + var ex = Assert.Throws(() => options.Validate("standard")); + Assert.Contains("lockout.windowMinutes", ex.Message, StringComparison.OrdinalIgnoreCase); + } + + [Fact] + public void Normalize_ResolvesRelativeTokenSigningDirectory() + { + var configDir = Path.Combine(Path.GetTempPath(), "stellaops-standard-plugin", Guid.NewGuid().ToString("N")); + Directory.CreateDirectory(configDir); + + try + { + var configPath = Path.Combine(configDir, "standard.yaml"); + var options = new StandardPluginOptions + { + TokenSigning = { KeyDirectory = "../keys" } + }; + + options.Normalize(configPath); + + var expected = Path.GetFullPath(Path.Combine(configDir, "../keys")); + Assert.Equal(expected, options.TokenSigning.KeyDirectory); + } + finally + { + if (Directory.Exists(configDir)) + { + Directory.Delete(configDir, recursive: true); + } + } + } + + [Fact] + public void Normalize_PreservesAbsoluteTokenSigningDirectory() + { + var absolute = Path.Combine(Path.GetTempPath(), "stellaops-standard-plugin", Guid.NewGuid().ToString("N"), "keys"); + var options = new StandardPluginOptions + { + TokenSigning = { KeyDirectory = absolute } + }; + + options.Normalize(Path.Combine(Path.GetTempPath(), "config", "standard.yaml")); + + Assert.Equal(Path.GetFullPath(absolute), options.TokenSigning.KeyDirectory); + } +} diff --git a/src/StellaOps.Authority/StellaOps.Authority.Plugin.Standard.Tests/StandardPluginRegistrarTests.cs b/src/StellaOps.Authority/StellaOps.Authority.Plugin.Standard.Tests/StandardPluginRegistrarTests.cs new file mode 100644 index 00000000..19a432e6 --- /dev/null +++ b/src/StellaOps.Authority/StellaOps.Authority.Plugin.Standard.Tests/StandardPluginRegistrarTests.cs @@ -0,0 +1,227 @@ +using System; +using System.Collections.Generic; +using System.IO; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.Extensions.Configuration; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Hosting; +using Microsoft.Extensions.Options; +using Mongo2Go; +using MongoDB.Driver; +using StellaOps.Authority.Plugins.Abstractions; +using StellaOps.Authority.Plugin.Standard; +using StellaOps.Authority.Plugin.Standard.Bootstrap; +using StellaOps.Authority.Plugin.Standard.Storage; +using StellaOps.Authority.Storage.Mongo.Documents; +using StellaOps.Authority.Storage.Mongo.Stores; + +namespace StellaOps.Authority.Plugin.Standard.Tests; + +public class StandardPluginRegistrarTests +{ + [Fact] + public async Task Register_ConfiguresIdentityProviderAndSeedsBootstrapUser() + { + using var runner = MongoDbRunner.Start(singleNodeReplSet: true); + var client = new MongoClient(runner.ConnectionString); + var database = client.GetDatabase("registrar-tests"); + + var configuration = new ConfigurationBuilder() + .AddInMemoryCollection(new Dictionary + { + ["passwordPolicy:minimumLength"] = "8", + ["passwordPolicy:requireDigit"] = "false", + ["passwordPolicy:requireSymbol"] = "false", + ["lockout:enabled"] = "false", + ["bootstrapUser:username"] = "bootstrap", + ["bootstrapUser:password"] = "Bootstrap1!", + ["bootstrapUser:requirePasswordReset"] = "true" + }) + .Build(); + + var manifest = new AuthorityPluginManifest( + "standard", + "standard", + true, + typeof(StandardPluginRegistrar).Assembly.GetName().Name, + typeof(StandardPluginRegistrar).Assembly.Location, + new[] { AuthorityPluginCapabilities.Password, AuthorityPluginCapabilities.Bootstrap, AuthorityPluginCapabilities.ClientProvisioning }, + new Dictionary(), + "standard.yaml"); + + var pluginContext = new AuthorityPluginContext(manifest, configuration); + var services = new ServiceCollection(); + services.AddLogging(); + services.AddSingleton(database); + services.AddSingleton(new InMemoryClientStore()); + + var registrar = new StandardPluginRegistrar(); + registrar.Register(new AuthorityPluginRegistrationContext(services, pluginContext, configuration)); + + var provider = services.BuildServiceProvider(); + var hostedServices = provider.GetServices(); + foreach (var hosted in hostedServices) + { + if (hosted is StandardPluginBootstrapper bootstrapper) + { + await bootstrapper.StartAsync(CancellationToken.None); + } + } + + var plugin = provider.GetRequiredService(); + Assert.Equal("standard", plugin.Type); + Assert.True(plugin.Capabilities.SupportsPassword); + Assert.False(plugin.Capabilities.SupportsMfa); + Assert.True(plugin.Capabilities.SupportsClientProvisioning); + + var verification = await plugin.Credentials.VerifyPasswordAsync("bootstrap", "Bootstrap1!", CancellationToken.None); + Assert.True(verification.Succeeded); + Assert.True(verification.User?.RequiresPasswordReset); + } + + [Fact] + public void Register_ForcesPasswordCapability_WhenManifestMissing() + { + using var runner = MongoDbRunner.Start(singleNodeReplSet: true); + var client = new MongoClient(runner.ConnectionString); + var database = client.GetDatabase("registrar-capabilities"); + + var configuration = new ConfigurationBuilder().Build(); + var manifest = new AuthorityPluginManifest( + "standard", + "standard", + true, + typeof(StandardPluginRegistrar).Assembly.GetName().Name, + typeof(StandardPluginRegistrar).Assembly.Location, + Array.Empty(), + new Dictionary(), + "standard.yaml"); + + var pluginContext = new AuthorityPluginContext(manifest, configuration); + var services = new ServiceCollection(); + services.AddLogging(); + services.AddSingleton(database); + services.AddSingleton(new InMemoryClientStore()); + + var registrar = new StandardPluginRegistrar(); + registrar.Register(new AuthorityPluginRegistrationContext(services, pluginContext, configuration)); + + using var provider = services.BuildServiceProvider(); + var plugin = provider.GetRequiredService(); + + Assert.True(plugin.Capabilities.SupportsPassword); + } + + [Fact] + public void Register_Throws_WhenBootstrapConfigurationIncomplete() + { + using var runner = MongoDbRunner.Start(singleNodeReplSet: true); + var client = new MongoClient(runner.ConnectionString); + var database = client.GetDatabase("registrar-bootstrap-validation"); + + var configuration = new ConfigurationBuilder() + .AddInMemoryCollection(new Dictionary + { + ["bootstrapUser:username"] = "bootstrap" + }) + .Build(); + + var manifest = new AuthorityPluginManifest( + "standard", + "standard", + true, + typeof(StandardPluginRegistrar).Assembly.GetName().Name, + typeof(StandardPluginRegistrar).Assembly.Location, + new[] { AuthorityPluginCapabilities.Password }, + new Dictionary(), + "standard.yaml"); + + var pluginContext = new AuthorityPluginContext(manifest, configuration); + var services = new ServiceCollection(); + services.AddLogging(); + services.AddSingleton(database); + services.AddSingleton(new InMemoryClientStore()); + + var registrar = new StandardPluginRegistrar(); + registrar.Register(new AuthorityPluginRegistrationContext(services, pluginContext, configuration)); + + using var provider = services.BuildServiceProvider(); + Assert.Throws(() => provider.GetRequiredService()); + } + + [Fact] + public void Register_NormalizesTokenSigningKeyDirectory() + { + using var runner = MongoDbRunner.Start(singleNodeReplSet: true); + var client = new MongoClient(runner.ConnectionString); + var database = client.GetDatabase("registrar-token-signing"); + + var configuration = new ConfigurationBuilder() + .AddInMemoryCollection(new Dictionary + { + ["tokenSigning:keyDirectory"] = "../keys" + }) + .Build(); + + var configDir = Path.Combine(Path.GetTempPath(), "stellaops-standard-plugin", Guid.NewGuid().ToString("N")); + Directory.CreateDirectory(configDir); + + try + { + var configPath = Path.Combine(configDir, "standard.yaml"); + var manifest = new AuthorityPluginManifest( + "standard", + "standard", + true, + typeof(StandardPluginRegistrar).Assembly.GetName().Name, + typeof(StandardPluginRegistrar).Assembly.Location, + new[] { AuthorityPluginCapabilities.Password }, + new Dictionary(), + configPath); + + var pluginContext = new AuthorityPluginContext(manifest, configuration); + var services = new ServiceCollection(); + services.AddLogging(); + services.AddSingleton(database); + services.AddSingleton(new InMemoryClientStore()); + + var registrar = new StandardPluginRegistrar(); + registrar.Register(new AuthorityPluginRegistrationContext(services, pluginContext, configuration)); + + using var provider = services.BuildServiceProvider(); + var optionsMonitor = provider.GetRequiredService>(); + var options = optionsMonitor.Get("standard"); + + var expected = Path.GetFullPath(Path.Combine(configDir, "../keys")); + Assert.Equal(expected, options.TokenSigning.KeyDirectory); + } + finally + { + if (Directory.Exists(configDir)) + { + Directory.Delete(configDir, recursive: true); + } + } + } +} + +internal sealed class InMemoryClientStore : IAuthorityClientStore +{ + private readonly Dictionary clients = new(StringComparer.OrdinalIgnoreCase); + + public ValueTask FindByClientIdAsync(string clientId, CancellationToken cancellationToken) + { + clients.TryGetValue(clientId, out var document); + return ValueTask.FromResult(document); + } + + public ValueTask UpsertAsync(AuthorityClientDocument document, CancellationToken cancellationToken) + { + clients[document.ClientId] = document; + return ValueTask.CompletedTask; + } + + public ValueTask DeleteByClientIdAsync(string clientId, CancellationToken cancellationToken) + => ValueTask.FromResult(clients.Remove(clientId)); +} diff --git a/src/StellaOps.Authority/StellaOps.Authority.Plugin.Standard.Tests/StandardUserCredentialStoreTests.cs b/src/StellaOps.Authority/StellaOps.Authority.Plugin.Standard.Tests/StandardUserCredentialStoreTests.cs new file mode 100644 index 00000000..92101303 --- /dev/null +++ b/src/StellaOps.Authority/StellaOps.Authority.Plugin.Standard.Tests/StandardUserCredentialStoreTests.cs @@ -0,0 +1,102 @@ +using System.Collections.Generic; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.Extensions.Logging.Abstractions; +using Mongo2Go; +using MongoDB.Driver; +using StellaOps.Authority.Plugins.Abstractions; +using StellaOps.Authority.Plugin.Standard.Security; +using StellaOps.Authority.Plugin.Standard.Storage; + +namespace StellaOps.Authority.Plugin.Standard.Tests; + +public class StandardUserCredentialStoreTests : IAsyncLifetime +{ + private readonly MongoDbRunner runner; + private readonly IMongoDatabase database; + private readonly StandardPluginOptions options; + private readonly StandardUserCredentialStore store; + + public StandardUserCredentialStoreTests() + { + runner = MongoDbRunner.Start(singleNodeReplSet: true); + var client = new MongoClient(runner.ConnectionString); + database = client.GetDatabase("authority-tests"); + options = new StandardPluginOptions + { + PasswordPolicy = new PasswordPolicyOptions + { + MinimumLength = 8, + RequireDigit = true, + RequireLowercase = true, + RequireUppercase = true, + RequireSymbol = false + }, + Lockout = new LockoutOptions + { + Enabled = true, + MaxAttempts = 2, + WindowMinutes = 1 + } + }; + store = new StandardUserCredentialStore( + "standard", + database, + options, + new Pbkdf2PasswordHasher(), + NullLogger.Instance); + } + + [Fact] + public async Task VerifyPasswordAsync_ReturnsSuccess_ForValidCredentials() + { + var registration = new AuthorityUserRegistration( + "alice", + "Password1!", + "Alice", + null, + false, + new[] { "admin" }, + new Dictionary()); + + var upsert = await store.UpsertUserAsync(registration, CancellationToken.None); + Assert.True(upsert.Succeeded); + + var result = await store.VerifyPasswordAsync("alice", "Password1!", CancellationToken.None); + Assert.True(result.Succeeded); + Assert.Equal("alice", result.User?.Username); + } + + [Fact] + public async Task VerifyPasswordAsync_EnforcesLockout_AfterRepeatedFailures() + { + await store.UpsertUserAsync( + new AuthorityUserRegistration( + "bob", + "Password1!", + "Bob", + null, + false, + new[] { "operator" }, + new Dictionary()), + CancellationToken.None); + + var first = await store.VerifyPasswordAsync("bob", "wrong", CancellationToken.None); + Assert.False(first.Succeeded); + Assert.Equal(AuthorityCredentialFailureCode.InvalidCredentials, first.FailureCode); + + var second = await store.VerifyPasswordAsync("bob", "stillwrong", CancellationToken.None); + Assert.False(second.Succeeded); + Assert.Equal(AuthorityCredentialFailureCode.LockedOut, second.FailureCode); + Assert.NotNull(second.RetryAfter); + Assert.True(second.RetryAfter.Value > System.TimeSpan.Zero); + } + + public Task InitializeAsync() => Task.CompletedTask; + + public Task DisposeAsync() + { + runner.Dispose(); + return Task.CompletedTask; + } +} diff --git a/src/StellaOps.Authority/StellaOps.Authority.Plugin.Standard.Tests/StellaOps.Authority.Plugin.Standard.Tests.csproj b/src/StellaOps.Authority/StellaOps.Authority.Plugin.Standard.Tests/StellaOps.Authority.Plugin.Standard.Tests.csproj new file mode 100644 index 00000000..e80ccc23 --- /dev/null +++ b/src/StellaOps.Authority/StellaOps.Authority.Plugin.Standard.Tests/StellaOps.Authority.Plugin.Standard.Tests.csproj @@ -0,0 +1,12 @@ + + + net10.0 + enable + enable + false + + + + + + diff --git a/src/StellaOps.Authority/StellaOps.Authority.Plugin.Standard/AGENTS.md b/src/StellaOps.Authority/StellaOps.Authority.Plugin.Standard/AGENTS.md new file mode 100644 index 00000000..5dc29805 --- /dev/null +++ b/src/StellaOps.Authority/StellaOps.Authority.Plugin.Standard/AGENTS.md @@ -0,0 +1,20 @@ +# Plugin Team Charter + +## Mission +Own the Mongo-backed Standard identity provider plug-in and shared Authority plug-in contracts. Deliver secure credential flows, configuration validation, and documentation that help other identity providers integrate cleanly. + +## Responsibilities +- Maintain `StellaOps.Authority.Plugin.Standard` and related test projects. +- Coordinate schema/option changes with Authority Core and Docs guilds. +- Ensure plugin options remain deterministic and offline-friendly. +- Surface open work on `TASKS.md`; update statuses (TODO/DOING/DONE/BLOCKED/REVIEW). + +## Key Paths +- `StandardPluginOptions` & registrar wiring +- `StandardUserCredentialStore` (Mongo persistence + lockouts) +- `docs/dev/31_AUTHORITY_PLUGIN_DEVELOPER_GUIDE.md` + +## Coordination +- Team 2 (Authority Core) for handler integration. +- Security Guild for password hashing, audit, revocation. +- Docs Guild for developer guide polish and diagrams. diff --git a/src/StellaOps.Authority/StellaOps.Authority.Plugin.Standard/Bootstrap/StandardPluginBootstrapper.cs b/src/StellaOps.Authority/StellaOps.Authority.Plugin.Standard/Bootstrap/StandardPluginBootstrapper.cs new file mode 100644 index 00000000..01608007 --- /dev/null +++ b/src/StellaOps.Authority/StellaOps.Authority.Plugin.Standard/Bootstrap/StandardPluginBootstrapper.cs @@ -0,0 +1,42 @@ +using System.Threading; +using System.Threading.Tasks; +using Microsoft.Extensions.Hosting; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Options; +using StellaOps.Authority.Plugin.Standard.Storage; + +namespace StellaOps.Authority.Plugin.Standard.Bootstrap; + +internal sealed class StandardPluginBootstrapper : IHostedService +{ + private readonly string pluginName; + private readonly IOptionsMonitor optionsMonitor; + private readonly StandardUserCredentialStore credentialStore; + private readonly ILogger logger; + + public StandardPluginBootstrapper( + string pluginName, + IOptionsMonitor optionsMonitor, + StandardUserCredentialStore credentialStore, + ILogger logger) + { + this.pluginName = pluginName; + this.optionsMonitor = optionsMonitor; + this.credentialStore = credentialStore; + this.logger = logger; + } + + public async Task StartAsync(CancellationToken cancellationToken) + { + var options = optionsMonitor.Get(pluginName); + if (options.BootstrapUser is null || !options.BootstrapUser.IsConfigured) + { + return; + } + + logger.LogInformation("Standard Authority plugin '{PluginName}' ensuring bootstrap user.", pluginName); + await credentialStore.EnsureBootstrapUserAsync(options.BootstrapUser, cancellationToken).ConfigureAwait(false); + } + + public Task StopAsync(CancellationToken cancellationToken) => Task.CompletedTask; +} diff --git a/src/StellaOps.Authority/StellaOps.Authority.Plugin.Standard/Properties/AssemblyInfo.cs b/src/StellaOps.Authority/StellaOps.Authority.Plugin.Standard/Properties/AssemblyInfo.cs new file mode 100644 index 00000000..93fe0516 --- /dev/null +++ b/src/StellaOps.Authority/StellaOps.Authority.Plugin.Standard/Properties/AssemblyInfo.cs @@ -0,0 +1,3 @@ +using System.Runtime.CompilerServices; + +[assembly: InternalsVisibleTo("StellaOps.Authority.Plugin.Standard.Tests")] diff --git a/src/StellaOps.Authority/StellaOps.Authority.Plugin.Standard/Security/IPasswordHasher.cs b/src/StellaOps.Authority/StellaOps.Authority.Plugin.Standard/Security/IPasswordHasher.cs new file mode 100644 index 00000000..7264e8c1 --- /dev/null +++ b/src/StellaOps.Authority/StellaOps.Authority.Plugin.Standard/Security/IPasswordHasher.cs @@ -0,0 +1,113 @@ +using System; +using System.Security.Cryptography; +using System.Text; + +namespace StellaOps.Authority.Plugin.Standard.Security; + +internal interface IPasswordHasher +{ + string Hash(string password); + + PasswordVerificationResult Verify(string password, string hashedPassword); +} + +internal enum PasswordVerificationResult +{ + Failed, + Success, + SuccessRehashNeeded +} + +internal sealed class Pbkdf2PasswordHasher : IPasswordHasher +{ + private const int SaltSize = 16; + private const int HashSize = 32; + private const int Iterations = 210_000; + private const string Header = "PBKDF2"; + + public string Hash(string password) + { + if (string.IsNullOrEmpty(password)) + { + throw new ArgumentException("Password is required.", nameof(password)); + } + + Span salt = stackalloc byte[SaltSize]; + RandomNumberGenerator.Fill(salt); + + Span hash = stackalloc byte[HashSize]; + var derived = Rfc2898DeriveBytes.Pbkdf2(password, salt.ToArray(), Iterations, HashAlgorithmName.SHA256, HashSize); + derived.CopyTo(hash); + + var payload = new byte[1 + SaltSize + HashSize]; + payload[0] = 0x01; // version + salt.CopyTo(payload.AsSpan(1)); + hash.CopyTo(payload.AsSpan(1 + SaltSize)); + + var builder = new StringBuilder(); + builder.Append(Header); + builder.Append('.'); + builder.Append(Iterations); + builder.Append('.'); + builder.Append(Convert.ToBase64String(payload)); + return builder.ToString(); + } + + public PasswordVerificationResult Verify(string password, string hashedPassword) + { + if (string.IsNullOrEmpty(password) || string.IsNullOrEmpty(hashedPassword)) + { + return PasswordVerificationResult.Failed; + } + + var parts = hashedPassword.Split('.', StringSplitOptions.RemoveEmptyEntries); + if (parts.Length != 3 || !string.Equals(parts[0], Header, StringComparison.Ordinal)) + { + return PasswordVerificationResult.Failed; + } + + if (!int.TryParse(parts[1], out var iterations)) + { + return PasswordVerificationResult.Failed; + } + + byte[] payload; + try + { + payload = Convert.FromBase64String(parts[2]); + } + catch (FormatException) + { + return PasswordVerificationResult.Failed; + } + + if (payload.Length != 1 + SaltSize + HashSize) + { + return PasswordVerificationResult.Failed; + } + + var version = payload[0]; + if (version != 0x01) + { + return PasswordVerificationResult.Failed; + } + + var salt = new byte[SaltSize]; + Array.Copy(payload, 1, salt, 0, SaltSize); + + var expectedHash = new byte[HashSize]; + Array.Copy(payload, 1 + SaltSize, expectedHash, 0, HashSize); + + var actualHash = Rfc2898DeriveBytes.Pbkdf2(password, salt, iterations, HashAlgorithmName.SHA256, HashSize); + + var success = CryptographicOperations.FixedTimeEquals(expectedHash, actualHash); + if (!success) + { + return PasswordVerificationResult.Failed; + } + + return iterations < Iterations + ? PasswordVerificationResult.SuccessRehashNeeded + : PasswordVerificationResult.Success; + } +} diff --git a/src/StellaOps.Authority/StellaOps.Authority.Plugin.Standard/StandardClaimsEnricher.cs b/src/StellaOps.Authority/StellaOps.Authority.Plugin.Standard/StandardClaimsEnricher.cs new file mode 100644 index 00000000..fa526e19 --- /dev/null +++ b/src/StellaOps.Authority/StellaOps.Authority.Plugin.Standard/StandardClaimsEnricher.cs @@ -0,0 +1,43 @@ +using System; +using System.Linq; +using System.Security.Claims; +using System.Threading; +using System.Threading.Tasks; +using StellaOps.Authority.Plugins.Abstractions; + +namespace StellaOps.Authority.Plugin.Standard; + +internal sealed class StandardClaimsEnricher : IClaimsEnricher +{ + public ValueTask EnrichAsync( + ClaimsIdentity identity, + AuthorityClaimsEnrichmentContext context, + CancellationToken cancellationToken) + { + if (identity is null) + { + throw new ArgumentNullException(nameof(identity)); + } + + if (context.User is { } user) + { + foreach (var role in user.Roles.Where(static r => !string.IsNullOrWhiteSpace(r))) + { + if (!identity.HasClaim(ClaimTypes.Role, role)) + { + identity.AddClaim(new Claim(ClaimTypes.Role, role)); + } + } + + foreach (var pair in user.Attributes) + { + if (!string.IsNullOrWhiteSpace(pair.Key) && !identity.HasClaim(pair.Key, pair.Value ?? string.Empty)) + { + identity.AddClaim(new Claim(pair.Key, pair.Value ?? string.Empty)); + } + } + } + + return ValueTask.CompletedTask; + } +} diff --git a/src/StellaOps.Authority/StellaOps.Authority.Plugin.Standard/StandardIdentityProviderPlugin.cs b/src/StellaOps.Authority/StellaOps.Authority.Plugin.Standard/StandardIdentityProviderPlugin.cs new file mode 100644 index 00000000..07ef9b48 --- /dev/null +++ b/src/StellaOps.Authority/StellaOps.Authority.Plugin.Standard/StandardIdentityProviderPlugin.cs @@ -0,0 +1,65 @@ +using System; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.Extensions.Logging; +using StellaOps.Authority.Plugins.Abstractions; +using StellaOps.Authority.Plugin.Standard.Storage; + +namespace StellaOps.Authority.Plugin.Standard; + +internal sealed class StandardIdentityProviderPlugin : IIdentityProviderPlugin +{ + private readonly ILogger logger; + + public StandardIdentityProviderPlugin( + AuthorityPluginContext context, + StandardUserCredentialStore credentialStore, + StandardClientProvisioningStore clientProvisioningStore, + IClaimsEnricher claimsEnricher, + ILogger logger) + { + Context = context ?? throw new ArgumentNullException(nameof(context)); + Credentials = credentialStore ?? throw new ArgumentNullException(nameof(credentialStore)); + ClientProvisioning = clientProvisioningStore ?? throw new ArgumentNullException(nameof(clientProvisioningStore)); + ClaimsEnricher = claimsEnricher ?? throw new ArgumentNullException(nameof(claimsEnricher)); + this.logger = logger ?? throw new ArgumentNullException(nameof(logger)); + + var manifestCapabilities = AuthorityIdentityProviderCapabilities.FromCapabilities(context.Manifest.Capabilities); + if (!manifestCapabilities.SupportsPassword) + { + this.logger.LogWarning( + "Standard Authority plugin '{PluginName}' manifest does not declare the 'password' capability. Forcing password support.", + Context.Manifest.Name); + } + + Capabilities = manifestCapabilities with { SupportsPassword = true }; + } + + public string Name => Context.Manifest.Name; + + public string Type => Context.Manifest.Type; + + public AuthorityPluginContext Context { get; } + + public IUserCredentialStore Credentials { get; } + + public IClaimsEnricher ClaimsEnricher { get; } + + public IClientProvisioningStore? ClientProvisioning { get; } + + public AuthorityIdentityProviderCapabilities Capabilities { get; } + + public async ValueTask CheckHealthAsync(CancellationToken cancellationToken) + { + try + { + var store = (StandardUserCredentialStore)Credentials; + return await store.CheckHealthAsync(cancellationToken).ConfigureAwait(false); + } + catch (Exception ex) + { + logger.LogError(ex, "Standard Authority plugin '{PluginName}' health check failed.", Name); + return AuthorityPluginHealthResult.Unavailable(ex.Message); + } + } +} diff --git a/src/StellaOps.Authority/StellaOps.Authority.Plugin.Standard/StandardPluginOptions.cs b/src/StellaOps.Authority/StellaOps.Authority.Plugin.Standard/StandardPluginOptions.cs new file mode 100644 index 00000000..27915982 --- /dev/null +++ b/src/StellaOps.Authority/StellaOps.Authority.Plugin.Standard/StandardPluginOptions.cs @@ -0,0 +1,130 @@ +using System; +using System.IO; + +namespace StellaOps.Authority.Plugin.Standard; + +internal sealed class StandardPluginOptions +{ + public BootstrapUserOptions? BootstrapUser { get; set; } + + public PasswordPolicyOptions PasswordPolicy { get; set; } = new(); + + public LockoutOptions Lockout { get; set; } = new(); + + public TokenSigningOptions TokenSigning { get; set; } = new(); + + public void Normalize(string configPath) + { + TokenSigning.Normalize(configPath); + } + + public void Validate(string pluginName) + { + BootstrapUser?.Validate(pluginName); + PasswordPolicy.Validate(pluginName); + Lockout.Validate(pluginName); + } +} + +internal sealed class BootstrapUserOptions +{ + public string? Username { get; set; } + + public string? Password { get; set; } + + public bool RequirePasswordReset { get; set; } = true; + + public bool IsConfigured => !string.IsNullOrWhiteSpace(Username) && !string.IsNullOrWhiteSpace(Password); + + public void Validate(string pluginName) + { + var hasUsername = !string.IsNullOrWhiteSpace(Username); + var hasPassword = !string.IsNullOrWhiteSpace(Password); + + if (hasUsername ^ hasPassword) + { + throw new InvalidOperationException($"Standard plugin '{pluginName}' requires both bootstrapUser.username and bootstrapUser.password when configuring a bootstrap user."); + } + } +} + +internal sealed class PasswordPolicyOptions +{ + public int MinimumLength { get; set; } = 12; + + public bool RequireUppercase { get; set; } = true; + + public bool RequireLowercase { get; set; } = true; + + public bool RequireDigit { get; set; } = true; + + public bool RequireSymbol { get; set; } = true; + + public void Validate(string pluginName) + { + if (MinimumLength <= 0) + { + throw new InvalidOperationException($"Standard plugin '{pluginName}' requires passwordPolicy.minimumLength to be greater than zero."); + } + } +} + +internal sealed class LockoutOptions +{ + public bool Enabled { get; set; } = true; + + public int MaxAttempts { get; set; } = 5; + + public int WindowMinutes { get; set; } = 15; + + public TimeSpan Window => TimeSpan.FromMinutes(WindowMinutes <= 0 ? 15 : WindowMinutes); + + public void Validate(string pluginName) + { + if (Enabled && MaxAttempts <= 0) + { + throw new InvalidOperationException($"Standard plugin '{pluginName}' requires lockout.maxAttempts to be greater than zero when lockout is enabled."); + } + + if (Enabled && WindowMinutes <= 0) + { + throw new InvalidOperationException($"Standard plugin '{pluginName}' requires lockout.windowMinutes to be greater than zero when lockout is enabled."); + } + } +} + +internal sealed class TokenSigningOptions +{ + public string? KeyDirectory { get; set; } + + public void Normalize(string configPath) + { + if (string.IsNullOrWhiteSpace(KeyDirectory)) + { + KeyDirectory = null; + return; + } + + var resolved = KeyDirectory.Trim(); + if (string.IsNullOrEmpty(resolved)) + { + KeyDirectory = null; + return; + } + + resolved = Environment.ExpandEnvironmentVariables(resolved); + + if (!Path.IsPathRooted(resolved)) + { + var baseDirectory = Path.GetDirectoryName(configPath); + if (string.IsNullOrEmpty(baseDirectory)) + { + baseDirectory = Directory.GetCurrentDirectory(); + } + + resolved = Path.Combine(baseDirectory, resolved); + } + + KeyDirectory = Path.GetFullPath(resolved); + } +} diff --git a/src/StellaOps.Authority/StellaOps.Authority.Plugin.Standard/StandardPluginRegistrar.cs b/src/StellaOps.Authority/StellaOps.Authority.Plugin.Standard/StandardPluginRegistrar.cs new file mode 100644 index 00000000..570ae2fa --- /dev/null +++ b/src/StellaOps.Authority/StellaOps.Authority.Plugin.Standard/StandardPluginRegistrar.cs @@ -0,0 +1,88 @@ +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.DependencyInjection.Extensions; +using Microsoft.Extensions.Hosting; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Options; +using MongoDB.Driver; +using StellaOps.Authority.Plugins.Abstractions; +using StellaOps.Authority.Plugin.Standard.Bootstrap; +using StellaOps.Authority.Plugin.Standard.Security; +using StellaOps.Authority.Plugin.Standard.Storage; +using StellaOps.Authority.Storage.Mongo.Stores; + +namespace StellaOps.Authority.Plugin.Standard; + +internal sealed class StandardPluginRegistrar : IAuthorityPluginRegistrar +{ + public string PluginType => "standard"; + + public void Register(AuthorityPluginRegistrationContext context) + { + if (context is null) + { + throw new ArgumentNullException(nameof(context)); + } + + var pluginName = context.Plugin.Manifest.Name; + + context.Services.TryAddSingleton(); + context.Services.AddSingleton(); + context.Services.AddSingleton(sp => sp.GetRequiredService()); + + var configPath = context.Plugin.Manifest.ConfigPath; + + context.Services.AddOptions(pluginName) + .Bind(context.Plugin.Configuration) + .PostConfigure(options => + { + options.Normalize(configPath); + options.Validate(pluginName); + }) + .ValidateOnStart(); + + context.Services.AddSingleton(sp => + { + var database = sp.GetRequiredService(); + var optionsMonitor = sp.GetRequiredService>(); + var pluginOptions = optionsMonitor.Get(pluginName); + var passwordHasher = sp.GetRequiredService(); + var loggerFactory = sp.GetRequiredService(); + + return new StandardUserCredentialStore( + pluginName, + database, + pluginOptions, + passwordHasher, + loggerFactory.CreateLogger()); + }); + + context.Services.AddSingleton(sp => + { + var clientStore = sp.GetRequiredService(); + return new StandardClientProvisioningStore(pluginName, clientStore); + }); + + context.Services.AddSingleton(sp => + { + var store = sp.GetRequiredService(); + var clientProvisioningStore = sp.GetRequiredService(); + var loggerFactory = sp.GetRequiredService(); + return new StandardIdentityProviderPlugin( + context.Plugin, + store, + clientProvisioningStore, + sp.GetRequiredService(), + loggerFactory.CreateLogger()); + }); + + context.Services.AddSingleton(sp => + sp.GetRequiredService()); + + context.Services.AddSingleton(sp => + new StandardPluginBootstrapper( + pluginName, + sp.GetRequiredService>(), + sp.GetRequiredService(), + sp.GetRequiredService>())); + } +} diff --git a/src/StellaOps.Authority/StellaOps.Authority.Plugin.Standard/StellaOps.Authority.Plugin.Standard.csproj b/src/StellaOps.Authority/StellaOps.Authority.Plugin.Standard/StellaOps.Authority.Plugin.Standard.csproj new file mode 100644 index 00000000..913c50e6 --- /dev/null +++ b/src/StellaOps.Authority/StellaOps.Authority.Plugin.Standard/StellaOps.Authority.Plugin.Standard.csproj @@ -0,0 +1,22 @@ + + + net10.0 + preview + enable + enable + true + true + + + + + + + + + + + + + + diff --git a/src/StellaOps.Authority/StellaOps.Authority.Plugin.Standard/Storage/StandardClientProvisioningStore.cs b/src/StellaOps.Authority/StellaOps.Authority.Plugin.Standard/Storage/StandardClientProvisioningStore.cs new file mode 100644 index 00000000..c93a1d8a --- /dev/null +++ b/src/StellaOps.Authority/StellaOps.Authority.Plugin.Standard/Storage/StandardClientProvisioningStore.cs @@ -0,0 +1,109 @@ +using System.Linq; +using StellaOps.Authority.Plugins.Abstractions; +using StellaOps.Authority.Storage.Mongo.Documents; +using StellaOps.Authority.Storage.Mongo.Stores; + +namespace StellaOps.Authority.Plugin.Standard.Storage; + +internal sealed class StandardClientProvisioningStore : IClientProvisioningStore +{ + private readonly string pluginName; + private readonly IAuthorityClientStore clientStore; + + public StandardClientProvisioningStore(string pluginName, IAuthorityClientStore clientStore) + { + this.pluginName = pluginName ?? throw new ArgumentNullException(nameof(pluginName)); + this.clientStore = clientStore ?? throw new ArgumentNullException(nameof(clientStore)); + } + + public async ValueTask> CreateOrUpdateAsync( + AuthorityClientRegistration registration, + CancellationToken cancellationToken) + { + ArgumentNullException.ThrowIfNull(registration); + + if (registration.Confidential && string.IsNullOrWhiteSpace(registration.ClientSecret)) + { + return AuthorityPluginOperationResult.Failure("secret_required", "Confidential clients require a client secret."); + } + + var document = await clientStore.FindByClientIdAsync(registration.ClientId, cancellationToken).ConfigureAwait(false) + ?? new AuthorityClientDocument { ClientId = registration.ClientId, CreatedAt = DateTimeOffset.UtcNow }; + + document.Plugin = pluginName; + document.ClientType = registration.Confidential ? "confidential" : "public"; + document.DisplayName = registration.DisplayName; + document.SecretHash = registration.Confidential && registration.ClientSecret is not null + ? AuthoritySecretHasher.ComputeHash(registration.ClientSecret) + : null; + + document.RedirectUris = registration.RedirectUris.Select(static uri => uri.ToString()).ToList(); + document.PostLogoutRedirectUris = registration.PostLogoutRedirectUris.Select(static uri => uri.ToString()).ToList(); + + document.Properties[AuthorityClientMetadataKeys.AllowedGrantTypes] = string.Join(" ", registration.AllowedGrantTypes); + document.Properties[AuthorityClientMetadataKeys.AllowedScopes] = string.Join(" ", registration.AllowedScopes); + document.Properties[AuthorityClientMetadataKeys.RedirectUris] = string.Join(" ", document.RedirectUris); + document.Properties[AuthorityClientMetadataKeys.PostLogoutRedirectUris] = string.Join(" ", document.PostLogoutRedirectUris); + + foreach (var (key, value) in registration.Properties) + { + document.Properties[key] = value; + } + + await clientStore.UpsertAsync(document, cancellationToken).ConfigureAwait(false); + + return AuthorityPluginOperationResult.Success(ToDescriptor(document)); + } + + public async ValueTask FindByClientIdAsync(string clientId, CancellationToken cancellationToken) + { + var document = await clientStore.FindByClientIdAsync(clientId, cancellationToken).ConfigureAwait(false); + return document is null ? null : ToDescriptor(document); + } + + public async ValueTask DeleteAsync(string clientId, CancellationToken cancellationToken) + { + var deleted = await clientStore.DeleteByClientIdAsync(clientId, cancellationToken).ConfigureAwait(false); + return deleted + ? AuthorityPluginOperationResult.Success() + : AuthorityPluginOperationResult.Failure("not_found", "Client was not found."); + } + + private static AuthorityClientDescriptor ToDescriptor(AuthorityClientDocument document) + { + var allowedGrantTypes = Split(document.Properties, AuthorityClientMetadataKeys.AllowedGrantTypes); + var allowedScopes = Split(document.Properties, AuthorityClientMetadataKeys.AllowedScopes); + + var redirectUris = document.RedirectUris + .Select(static value => Uri.TryCreate(value, UriKind.Absolute, out var uri) ? uri : null) + .Where(static uri => uri is not null) + .Cast() + .ToArray(); + + var postLogoutUris = document.PostLogoutRedirectUris + .Select(static value => Uri.TryCreate(value, UriKind.Absolute, out var uri) ? uri : null) + .Where(static uri => uri is not null) + .Cast() + .ToArray(); + + return new AuthorityClientDescriptor( + document.ClientId, + document.DisplayName, + string.Equals(document.ClientType, "confidential", StringComparison.OrdinalIgnoreCase), + allowedGrantTypes, + allowedScopes, + redirectUris, + postLogoutUris, + document.Properties); + } + + private static IReadOnlyCollection Split(IReadOnlyDictionary properties, string key) + { + if (!properties.TryGetValue(key, out var value) || string.IsNullOrWhiteSpace(value)) + { + return Array.Empty(); + } + + return value.Split(' ', StringSplitOptions.RemoveEmptyEntries | StringSplitOptions.TrimEntries); + } +} diff --git a/src/StellaOps.Authority/StellaOps.Authority.Plugin.Standard/Storage/StandardUserCredentialStore.cs b/src/StellaOps.Authority/StellaOps.Authority.Plugin.Standard/Storage/StandardUserCredentialStore.cs new file mode 100644 index 00000000..8724f350 --- /dev/null +++ b/src/StellaOps.Authority/StellaOps.Authority.Plugin.Standard/Storage/StandardUserCredentialStore.cs @@ -0,0 +1,329 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.Extensions.Logging; +using MongoDB.Bson; +using MongoDB.Driver; +using StellaOps.Authority.Plugins.Abstractions; +using StellaOps.Authority.Plugin.Standard.Security; + +namespace StellaOps.Authority.Plugin.Standard.Storage; + +internal sealed class StandardUserCredentialStore : IUserCredentialStore +{ + private readonly IMongoCollection users; + private readonly StandardPluginOptions options; + private readonly IPasswordHasher passwordHasher; + private readonly ILogger logger; + private readonly string pluginName; + + public StandardUserCredentialStore( + string pluginName, + IMongoDatabase database, + StandardPluginOptions options, + IPasswordHasher passwordHasher, + ILogger logger) + { + this.pluginName = pluginName ?? throw new ArgumentNullException(nameof(pluginName)); + this.options = options ?? throw new ArgumentNullException(nameof(options)); + this.passwordHasher = passwordHasher ?? throw new ArgumentNullException(nameof(passwordHasher)); + this.logger = logger ?? throw new ArgumentNullException(nameof(logger)); + + ArgumentNullException.ThrowIfNull(database); + + var collectionName = $"authority_users_{pluginName.ToLowerInvariant()}"; + users = database.GetCollection(collectionName); + EnsureIndexes(); + } + + public async ValueTask VerifyPasswordAsync( + string username, + string password, + CancellationToken cancellationToken) + { + if (string.IsNullOrWhiteSpace(username) || string.IsNullOrEmpty(password)) + { + return AuthorityCredentialVerificationResult.Failure(AuthorityCredentialFailureCode.InvalidCredentials); + } + + var normalized = NormalizeUsername(username); + var user = await users.Find(u => u.NormalizedUsername == normalized) + .FirstOrDefaultAsync(cancellationToken) + .ConfigureAwait(false); + + if (user is null) + { + logger.LogWarning("Plugin {PluginName} failed password verification for unknown user {Username}.", pluginName, normalized); + return AuthorityCredentialVerificationResult.Failure(AuthorityCredentialFailureCode.InvalidCredentials); + } + + if (options.Lockout.Enabled && user.Lockout.LockoutEnd is { } lockoutEnd && lockoutEnd > DateTimeOffset.UtcNow) + { + var retryAfter = lockoutEnd - DateTimeOffset.UtcNow; + logger.LogWarning("Plugin {PluginName} denied access for {Username} due to lockout (retry after {RetryAfter}).", pluginName, normalized, retryAfter); + return AuthorityCredentialVerificationResult.Failure( + AuthorityCredentialFailureCode.LockedOut, + "Account is temporarily locked.", + retryAfter); + } + + var verification = passwordHasher.Verify(password, user.PasswordHash); + if (verification is PasswordVerificationResult.Success or PasswordVerificationResult.SuccessRehashNeeded) + { + if (verification == PasswordVerificationResult.SuccessRehashNeeded) + { + user.PasswordHash = passwordHasher.Hash(password); + } + + ResetLockout(user); + user.UpdatedAt = DateTimeOffset.UtcNow; + await users.ReplaceOneAsync( + Builders.Filter.Eq(u => u.Id, user.Id), + user, + cancellationToken: cancellationToken).ConfigureAwait(false); + + var descriptor = ToDescriptor(user); + return AuthorityCredentialVerificationResult.Success(descriptor, descriptor.RequiresPasswordReset ? "Password reset required." : null); + } + + await RegisterFailureAsync(user, cancellationToken).ConfigureAwait(false); + + var code = options.Lockout.Enabled && user.Lockout.LockoutEnd is { } lockout + ? AuthorityCredentialFailureCode.LockedOut + : AuthorityCredentialFailureCode.InvalidCredentials; + + TimeSpan? retry = user.Lockout.LockoutEnd is { } lockoutTime && lockoutTime > DateTimeOffset.UtcNow + ? lockoutTime - DateTimeOffset.UtcNow + : null; + + return AuthorityCredentialVerificationResult.Failure( + code, + code == AuthorityCredentialFailureCode.LockedOut ? "Account is temporarily locked." : "Invalid credentials.", + retry); + } + + public async ValueTask> UpsertUserAsync( + AuthorityUserRegistration registration, + CancellationToken cancellationToken) + { + ArgumentNullException.ThrowIfNull(registration); + + var normalized = NormalizeUsername(registration.Username); + var now = DateTimeOffset.UtcNow; + + if (!string.IsNullOrEmpty(registration.Password)) + { + var passwordValidation = ValidatePassword(registration.Password); + if (passwordValidation is not null) + { + return AuthorityPluginOperationResult.Failure("password_policy_violation", passwordValidation); + } + } + + var existing = await users.Find(u => u.NormalizedUsername == normalized) + .FirstOrDefaultAsync(cancellationToken) + .ConfigureAwait(false); + + if (existing is null) + { + if (string.IsNullOrEmpty(registration.Password)) + { + return AuthorityPluginOperationResult.Failure("password_required", "New users require a password."); + } + + var document = new StandardUserDocument + { + Username = registration.Username, + NormalizedUsername = normalized, + DisplayName = registration.DisplayName, + Email = registration.Email, + PasswordHash = passwordHasher.Hash(registration.Password!), + RequirePasswordReset = registration.RequirePasswordReset, + Roles = registration.Roles.ToList(), + Attributes = new Dictionary(registration.Attributes, StringComparer.OrdinalIgnoreCase), + CreatedAt = now, + UpdatedAt = now + }; + + await users.InsertOneAsync(document, cancellationToken: cancellationToken).ConfigureAwait(false); + return AuthorityPluginOperationResult.Success(ToDescriptor(document)); + } + + existing.Username = registration.Username; + existing.DisplayName = registration.DisplayName ?? existing.DisplayName; + existing.Email = registration.Email ?? existing.Email; + existing.Roles = registration.Roles.Any() + ? registration.Roles.ToList() + : existing.Roles; + + if (registration.Attributes.Count > 0) + { + foreach (var pair in registration.Attributes) + { + existing.Attributes[pair.Key] = pair.Value; + } + } + + if (!string.IsNullOrEmpty(registration.Password)) + { + existing.PasswordHash = passwordHasher.Hash(registration.Password!); + existing.RequirePasswordReset = registration.RequirePasswordReset; + } + else if (registration.RequirePasswordReset) + { + existing.RequirePasswordReset = true; + } + + existing.UpdatedAt = now; + + await users.ReplaceOneAsync( + Builders.Filter.Eq(u => u.Id, existing.Id), + existing, + cancellationToken: cancellationToken).ConfigureAwait(false); + + return AuthorityPluginOperationResult.Success(ToDescriptor(existing)); + } + + public async ValueTask FindBySubjectAsync(string subjectId, CancellationToken cancellationToken) + { + if (string.IsNullOrWhiteSpace(subjectId)) + { + return null; + } + + var user = await users.Find(u => u.SubjectId == subjectId) + .FirstOrDefaultAsync(cancellationToken) + .ConfigureAwait(false); + + return user is null ? null : ToDescriptor(user); + } + + public async Task EnsureBootstrapUserAsync(BootstrapUserOptions bootstrap, CancellationToken cancellationToken) + { + if (bootstrap is null || !bootstrap.IsConfigured) + { + return; + } + + var registration = new AuthorityUserRegistration( + bootstrap.Username!, + bootstrap.Password, + displayName: bootstrap.Username, + email: null, + requirePasswordReset: bootstrap.RequirePasswordReset, + roles: Array.Empty(), + attributes: new Dictionary(StringComparer.OrdinalIgnoreCase)); + + var result = await UpsertUserAsync(registration, cancellationToken).ConfigureAwait(false); + if (!result.Succeeded) + { + logger.LogWarning( + "Plugin {PluginName} failed to seed bootstrap user '{Username}': {Reason}", + pluginName, + bootstrap.Username, + result.ErrorCode); + } + } + + public async Task CheckHealthAsync(CancellationToken cancellationToken) + { + try + { + var command = new BsonDocument("ping", 1); + await users.Database.RunCommandAsync(command, cancellationToken: cancellationToken).ConfigureAwait(false); + return AuthorityPluginHealthResult.Healthy(); + } + catch (Exception ex) + { + logger.LogError(ex, "Plugin {PluginName} failed MongoDB health check.", pluginName); + return AuthorityPluginHealthResult.Unavailable(ex.Message); + } + } + + private string? ValidatePassword(string password) + { + if (password.Length < options.PasswordPolicy.MinimumLength) + { + return $"Password must be at least {options.PasswordPolicy.MinimumLength} characters long."; + } + + if (options.PasswordPolicy.RequireUppercase && !password.Any(char.IsUpper)) + { + return "Password must contain an uppercase letter."; + } + + if (options.PasswordPolicy.RequireLowercase && !password.Any(char.IsLower)) + { + return "Password must contain a lowercase letter."; + } + + if (options.PasswordPolicy.RequireDigit && !password.Any(char.IsDigit)) + { + return "Password must contain a digit."; + } + + if (options.PasswordPolicy.RequireSymbol && password.All(char.IsLetterOrDigit)) + { + return "Password must contain a symbol."; + } + + return null; + } + + private async Task RegisterFailureAsync(StandardUserDocument user, CancellationToken cancellationToken) + { + user.Lockout.LastFailure = DateTimeOffset.UtcNow; + user.Lockout.FailedAttempts += 1; + + if (options.Lockout.Enabled && user.Lockout.FailedAttempts >= options.Lockout.MaxAttempts) + { + user.Lockout.LockoutEnd = DateTimeOffset.UtcNow + options.Lockout.Window; + user.Lockout.FailedAttempts = 0; + } + + await users.ReplaceOneAsync( + Builders.Filter.Eq(u => u.Id, user.Id), + user, + cancellationToken: cancellationToken).ConfigureAwait(false); + } + + private static void ResetLockout(StandardUserDocument user) + { + user.Lockout.FailedAttempts = 0; + user.Lockout.LockoutEnd = null; + user.Lockout.LastFailure = null; + } + + private static string NormalizeUsername(string username) + => username.Trim().ToLowerInvariant(); + + private AuthorityUserDescriptor ToDescriptor(StandardUserDocument document) + => new( + document.SubjectId, + document.Username, + document.DisplayName, + document.RequirePasswordReset, + document.Roles, + document.Attributes); + + private void EnsureIndexes() + { + var indexKeys = Builders.IndexKeys + .Ascending(u => u.NormalizedUsername); + + var indexModel = new CreateIndexModel( + indexKeys, + new CreateIndexOptions { Unique = true, Name = "idx_normalized_username" }); + + try + { + users.Indexes.CreateOne(indexModel); + } + catch (MongoCommandException ex) when (ex.CodeName.Equals("IndexOptionsConflict", StringComparison.OrdinalIgnoreCase)) + { + logger.LogDebug("Plugin {PluginName} skipped index creation due to existing index.", pluginName); + } + } +} diff --git a/src/StellaOps.Authority/StellaOps.Authority.Plugin.Standard/Storage/StandardUserDocument.cs b/src/StellaOps.Authority/StellaOps.Authority.Plugin.Standard/Storage/StandardUserDocument.cs new file mode 100644 index 00000000..1ebdbf43 --- /dev/null +++ b/src/StellaOps.Authority/StellaOps.Authority.Plugin.Standard/Storage/StandardUserDocument.cs @@ -0,0 +1,64 @@ +using System; +using System.Collections.Generic; +using MongoDB.Bson; +using MongoDB.Bson.Serialization.Attributes; + +namespace StellaOps.Authority.Plugin.Standard.Storage; + +internal sealed class StandardUserDocument +{ + [BsonId] + public ObjectId Id { get; set; } + + [BsonElement("subjectId")] + public string SubjectId { get; set; } = Guid.NewGuid().ToString("N"); + + [BsonElement("username")] + public string Username { get; set; } = string.Empty; + + [BsonElement("normalizedUsername")] + public string NormalizedUsername { get; set; } = string.Empty; + + [BsonElement("passwordHash")] + public string PasswordHash { get; set; } = string.Empty; + + [BsonElement("displayName")] + [BsonIgnoreIfNull] + public string? DisplayName { get; set; } + + [BsonElement("email")] + [BsonIgnoreIfNull] + public string? Email { get; set; } + + [BsonElement("requirePasswordReset")] + public bool RequirePasswordReset { get; set; } + + [BsonElement("roles")] + public List Roles { get; set; } = new(); + + [BsonElement("attributes")] + public Dictionary Attributes { get; set; } = new(StringComparer.OrdinalIgnoreCase); + + [BsonElement("lockout")] + public StandardLockoutState Lockout { get; set; } = new(); + + [BsonElement("createdAt")] + public DateTimeOffset CreatedAt { get; set; } = DateTimeOffset.UtcNow; + + [BsonElement("updatedAt")] + public DateTimeOffset UpdatedAt { get; set; } = DateTimeOffset.UtcNow; +} + +internal sealed class StandardLockoutState +{ + [BsonElement("failedAttempts")] + public int FailedAttempts { get; set; } + + [BsonElement("lockoutEnd")] + [BsonIgnoreIfNull] + public DateTimeOffset? LockoutEnd { get; set; } + + [BsonElement("lastFailure")] + [BsonIgnoreIfNull] + public DateTimeOffset? LastFailure { get; set; } +} diff --git a/src/StellaOps.Authority/StellaOps.Authority.Plugin.Standard/TASKS.md b/src/StellaOps.Authority/StellaOps.Authority.Plugin.Standard/TASKS.md new file mode 100644 index 00000000..4a53b9bb --- /dev/null +++ b/src/StellaOps.Authority/StellaOps.Authority.Plugin.Standard/TASKS.md @@ -0,0 +1,16 @@ +# Team 8 / Plugin Standard Backlog (UTC 2025-10-10) + +| ID | Status | Owner(s) | Depends on | Description | Exit Criteria | +|----|--------|----------|------------|-------------|---------------| +| PLG6.DOC | DONE (2025-10-11) | BE-Auth Plugin, Docs Guild | PLG1–PLG5 | Final polish + diagrams for plugin developer guide. | Docs team delivers copy-edit + exported diagrams; PR merged. | +| SEC1.PLG | TODO | Security Guild, BE-Auth Plugin | SEC1.A (StellaOps.Cryptography) | Swap Standard plugin hashing to Argon2id via `StellaOps.Cryptography` abstractions; keep PBKDF2 verification for legacy. | ✅ `StandardUserCredentialStore` uses `ICryptoProvider` to hash/check; ✅ Transparent rehash on success; ✅ Unit tests cover tamper + legacy rehash. | +| SEC1.OPT | TODO | Security Guild | SEC1.PLG | Expose password hashing knobs in `StandardPluginOptions` (`memoryKiB`, `iterations`, `parallelism`, `algorithm`) with validation. | ✅ Options bound from YAML; ✅ Invalid configs throw; ✅ Docs include tuning guidance. | +| SEC2.PLG | TODO | Security Guild, Storage Guild | SEC2.A (audit contract) | Emit audit events from password verification outcomes and persist via `IAuthorityLoginAttemptStore`. | ✅ Serilog events enriched with subject/client/IP/outcome; ✅ Mongo records written per attempt; ✅ Tests assert success/lockout/failure cases. | +| SEC3.PLG | TODO | Security Guild, BE-Auth Plugin | CORE8, SEC3.A (rate limiter) | Ensure lockout responses and rate-limit metadata flow through plugin logs/events (include retry-after). | ✅ Audit record includes retry-after; ✅ Tests confirm lockout + limiter interplay. | +| SEC4.PLG | TODO | Security Guild | SEC4.A (revocation schema) | Provide plugin hooks so revoked users/clients write reasons for revocation bundle export. | ✅ Revocation exporter consumes plugin data; ✅ Tests cover revoked user/client output. | +| SEC5.PLG | TODO | Security Guild | SEC5.A (threat model) | Address plugin-specific mitigations (bootstrap user handling, password policy docs) in threat model backlog. | ✅ Threat model lists plugin attack surfaces; ✅ Mitigation items filed. | +| PLG4-6.CAPABILITIES | DOING (2025-10-10) | BE-Auth Plugin, Docs Guild | PLG1–PLG3 | Finalise capability metadata exposure, config validation, and developer guide updates; remaining action is Docs polish/diagram export. | ✅ Capability metadata + validation merged; ✅ Plugin guide updated with final copy & diagrams; ✅ Release notes mention new toggles. | +| PLG7.RFC | REVIEW | BE-Auth Plugin, Security Guild | PLG4 | Socialize LDAP plugin RFC (`docs/rfcs/authority-plugin-ldap.md`) and capture guild feedback. | ✅ Guild review sign-off recorded; ✅ Follow-up issues filed in module boards. | +| PLG6.DIAGRAM | TODO | Docs Guild | PLG6.DOC | Export final sequence/component diagrams for the developer guide and add offline-friendly assets under `docs/assets/authority`. | ✅ Mermaid sources committed; ✅ Rendered SVG/PNG linked from Section 2 + Section 9; ✅ Docs build preview shared with Plugin + Docs guilds. | + +> Update statuses to DOING/DONE/BLOCKED as you make progress. Always run `dotnet test` for touched projects before marking DONE. diff --git a/src/StellaOps.Authority/StellaOps.Authority.Plugins.Abstractions.Tests/AuthorityClientRegistrationTests.cs b/src/StellaOps.Authority/StellaOps.Authority.Plugins.Abstractions.Tests/AuthorityClientRegistrationTests.cs new file mode 100644 index 00000000..10c2f18c --- /dev/null +++ b/src/StellaOps.Authority/StellaOps.Authority.Plugins.Abstractions.Tests/AuthorityClientRegistrationTests.cs @@ -0,0 +1,31 @@ +using System; +using StellaOps.Authority.Plugins.Abstractions; + +namespace StellaOps.Authority.Plugins.Abstractions.Tests; + +public class AuthorityClientRegistrationTests +{ + [Fact] + public void Constructor_Throws_WhenClientIdMissing() + { + Assert.Throws(() => new AuthorityClientRegistration(string.Empty, false, null, null)); + } + + [Fact] + public void Constructor_RequiresSecret_ForConfidentialClients() + { + Assert.Throws(() => new AuthorityClientRegistration("cli", true, null, null)); + } + + [Fact] + public void WithClientSecret_ReturnsCopy() + { + var registration = new AuthorityClientRegistration("cli", false, null, null); + + var updated = registration.WithClientSecret("secret"); + + Assert.Equal("cli", updated.ClientId); + Assert.Equal("secret", updated.ClientSecret); + Assert.False(updated.Confidential); + } +} diff --git a/src/StellaOps.Authority/StellaOps.Authority.Plugins.Abstractions.Tests/AuthorityCredentialVerificationResultTests.cs b/src/StellaOps.Authority/StellaOps.Authority.Plugins.Abstractions.Tests/AuthorityCredentialVerificationResultTests.cs new file mode 100644 index 00000000..97f65d98 --- /dev/null +++ b/src/StellaOps.Authority/StellaOps.Authority.Plugins.Abstractions.Tests/AuthorityCredentialVerificationResultTests.cs @@ -0,0 +1,38 @@ +using System; +using StellaOps.Authority.Plugins.Abstractions; + +namespace StellaOps.Authority.Plugins.Abstractions.Tests; + +public class AuthorityCredentialVerificationResultTests +{ + [Fact] + public void Success_SetsUserAndClearsFailure() + { + var user = new AuthorityUserDescriptor("subject-1", "user", "User", false); + + var result = AuthorityCredentialVerificationResult.Success(user, "ok"); + + Assert.True(result.Succeeded); + Assert.Equal(user, result.User); + Assert.Null(result.FailureCode); + Assert.Equal("ok", result.Message); + } + + [Fact] + public void Success_Throws_WhenUserNull() + { + Assert.Throws(() => AuthorityCredentialVerificationResult.Success(null!)); + } + + [Fact] + public void Failure_SetsFailureCode() + { + var result = AuthorityCredentialVerificationResult.Failure(AuthorityCredentialFailureCode.LockedOut, "locked", TimeSpan.FromMinutes(5)); + + Assert.False(result.Succeeded); + Assert.Null(result.User); + Assert.Equal(AuthorityCredentialFailureCode.LockedOut, result.FailureCode); + Assert.Equal("locked", result.Message); + Assert.Equal(TimeSpan.FromMinutes(5), result.RetryAfter); + } +} diff --git a/src/StellaOps.Authority/StellaOps.Authority.Plugins.Abstractions.Tests/AuthorityIdentityProviderCapabilitiesTests.cs b/src/StellaOps.Authority/StellaOps.Authority.Plugins.Abstractions.Tests/AuthorityIdentityProviderCapabilitiesTests.cs new file mode 100644 index 00000000..f73835f7 --- /dev/null +++ b/src/StellaOps.Authority/StellaOps.Authority.Plugins.Abstractions.Tests/AuthorityIdentityProviderCapabilitiesTests.cs @@ -0,0 +1,42 @@ +using System; +using StellaOps.Authority.Plugins.Abstractions; + +namespace StellaOps.Authority.Plugins.Abstractions.Tests; + +public class AuthorityIdentityProviderCapabilitiesTests +{ + [Fact] + public void FromCapabilities_SetsFlags_WhenTokensPresent() + { + var capabilities = AuthorityIdentityProviderCapabilities.FromCapabilities(new[] + { + "password", + "mfa", + "clientProvisioning" + }); + + Assert.True(capabilities.SupportsPassword); + Assert.True(capabilities.SupportsMfa); + Assert.True(capabilities.SupportsClientProvisioning); + } + + [Fact] + public void FromCapabilities_DefaultsToFalse_WhenEmpty() + { + var capabilities = AuthorityIdentityProviderCapabilities.FromCapabilities(Array.Empty()); + + Assert.False(capabilities.SupportsPassword); + Assert.False(capabilities.SupportsMfa); + Assert.False(capabilities.SupportsClientProvisioning); + } + + [Fact] + public void FromCapabilities_IgnoresNullSet() + { + var capabilities = AuthorityIdentityProviderCapabilities.FromCapabilities(null!); + + Assert.False(capabilities.SupportsPassword); + Assert.False(capabilities.SupportsMfa); + Assert.False(capabilities.SupportsClientProvisioning); + } +} diff --git a/src/StellaOps.Authority/StellaOps.Authority.Plugins.Abstractions.Tests/AuthorityPluginHealthResultTests.cs b/src/StellaOps.Authority/StellaOps.Authority.Plugins.Abstractions.Tests/AuthorityPluginHealthResultTests.cs new file mode 100644 index 00000000..a0117912 --- /dev/null +++ b/src/StellaOps.Authority/StellaOps.Authority.Plugins.Abstractions.Tests/AuthorityPluginHealthResultTests.cs @@ -0,0 +1,32 @@ +using StellaOps.Authority.Plugins.Abstractions; + +namespace StellaOps.Authority.Plugins.Abstractions.Tests; + +public class AuthorityPluginHealthResultTests +{ + [Fact] + public void Healthy_ReturnsHealthyStatus() + { + var result = AuthorityPluginHealthResult.Healthy("ready"); + + Assert.Equal(AuthorityPluginHealthStatus.Healthy, result.Status); + Assert.Equal("ready", result.Message); + Assert.NotNull(result.Details); + } + + [Fact] + public void Degraded_ReturnsDegradedStatus() + { + var result = AuthorityPluginHealthResult.Degraded("slow"); + + Assert.Equal(AuthorityPluginHealthStatus.Degraded, result.Status); + } + + [Fact] + public void Unavailable_ReturnsUnavailableStatus() + { + var result = AuthorityPluginHealthResult.Unavailable("down"); + + Assert.Equal(AuthorityPluginHealthStatus.Unavailable, result.Status); + } +} diff --git a/src/StellaOps.Authority/StellaOps.Authority.Plugins.Abstractions.Tests/AuthorityPluginOperationResultTests.cs b/src/StellaOps.Authority/StellaOps.Authority.Plugins.Abstractions.Tests/AuthorityPluginOperationResultTests.cs new file mode 100644 index 00000000..1d3541b5 --- /dev/null +++ b/src/StellaOps.Authority/StellaOps.Authority.Plugins.Abstractions.Tests/AuthorityPluginOperationResultTests.cs @@ -0,0 +1,60 @@ +using System; +using StellaOps.Authority.Plugins.Abstractions; + +namespace StellaOps.Authority.Plugins.Abstractions.Tests; + +public class AuthorityPluginOperationResultTests +{ + [Fact] + public void Success_ReturnsSucceededResult() + { + var result = AuthorityPluginOperationResult.Success("ok"); + + Assert.True(result.Succeeded); + Assert.Null(result.ErrorCode); + Assert.Equal("ok", result.Message); + } + + [Fact] + public void Failure_PopulatesErrorCode() + { + var result = AuthorityPluginOperationResult.Failure("ERR_CODE", "failure"); + + Assert.False(result.Succeeded); + Assert.Equal("ERR_CODE", result.ErrorCode); + Assert.Equal("failure", result.Message); + } + + [Fact] + public void Failure_Throws_WhenErrorCodeMissing() + { + Assert.Throws(() => AuthorityPluginOperationResult.Failure(string.Empty)); + } + + [Fact] + public void GenericSuccess_ReturnsValue() + { + var result = AuthorityPluginOperationResult.Success("value", "created"); + + Assert.True(result.Succeeded); + Assert.Equal("value", result.Value); + Assert.Equal("created", result.Message); + } + + [Fact] + public void GenericFailure_PopulatesErrorCode() + { + var result = AuthorityPluginOperationResult.Failure("CONFLICT", "duplicate"); + + Assert.False(result.Succeeded); + Assert.Equal(default, result.Value); + Assert.Equal("CONFLICT", result.ErrorCode); + Assert.Equal("duplicate", result.Message); + } + + [Fact] + public void GenericFailure_Throws_WhenErrorCodeMissing() + { + Assert.Throws(() => AuthorityPluginOperationResult.Failure(" ")); + } +} diff --git a/src/StellaOps.Authority/StellaOps.Authority.Plugins.Abstractions.Tests/AuthorityUserDescriptorTests.cs b/src/StellaOps.Authority/StellaOps.Authority.Plugins.Abstractions.Tests/AuthorityUserDescriptorTests.cs new file mode 100644 index 00000000..947e72cc --- /dev/null +++ b/src/StellaOps.Authority/StellaOps.Authority.Plugins.Abstractions.Tests/AuthorityUserDescriptorTests.cs @@ -0,0 +1,28 @@ +using System; +using StellaOps.Authority.Plugins.Abstractions; + +namespace StellaOps.Authority.Plugins.Abstractions.Tests; + +public class AuthorityUserDescriptorTests +{ + [Fact] + public void Constructor_Throws_WhenSubjectMissing() + { + Assert.Throws(() => new AuthorityUserDescriptor(string.Empty, "user", null, false)); + } + + [Fact] + public void Constructor_Throws_WhenUsernameMissing() + { + Assert.Throws(() => new AuthorityUserDescriptor("subject", " ", null, false)); + } + + [Fact] + public void Constructor_MaterialisesCollections() + { + var descriptor = new AuthorityUserDescriptor("subject", "user", null, false); + + Assert.NotNull(descriptor.Roles); + Assert.NotNull(descriptor.Attributes); + } +} diff --git a/src/StellaOps.Authority/StellaOps.Authority.Plugins.Abstractions.Tests/AuthorityUserRegistrationTests.cs b/src/StellaOps.Authority/StellaOps.Authority.Plugins.Abstractions.Tests/AuthorityUserRegistrationTests.cs new file mode 100644 index 00000000..12954946 --- /dev/null +++ b/src/StellaOps.Authority/StellaOps.Authority.Plugins.Abstractions.Tests/AuthorityUserRegistrationTests.cs @@ -0,0 +1,25 @@ +using System; +using StellaOps.Authority.Plugins.Abstractions; + +namespace StellaOps.Authority.Plugins.Abstractions.Tests; + +public class AuthorityUserRegistrationTests +{ + [Fact] + public void Constructor_Throws_WhenUsernameMissing() + { + Assert.Throws(() => new AuthorityUserRegistration(string.Empty, null, null, null, false)); + } + + [Fact] + public void WithPassword_ReturnsCopyWithPassword() + { + var registration = new AuthorityUserRegistration("alice", null, "Alice", null, true); + + var updated = registration.WithPassword("secret"); + + Assert.Equal("alice", updated.Username); + Assert.Equal("secret", updated.Password); + Assert.True(updated.RequirePasswordReset); + } +} diff --git a/src/StellaOps.Authority/StellaOps.Authority.Plugins.Abstractions.Tests/StellaOps.Authority.Plugins.Abstractions.Tests.csproj b/src/StellaOps.Authority/StellaOps.Authority.Plugins.Abstractions.Tests/StellaOps.Authority.Plugins.Abstractions.Tests.csproj new file mode 100644 index 00000000..b3d3ace1 --- /dev/null +++ b/src/StellaOps.Authority/StellaOps.Authority.Plugins.Abstractions.Tests/StellaOps.Authority.Plugins.Abstractions.Tests.csproj @@ -0,0 +1,11 @@ + + + net10.0 + enable + enable + false + + + + + diff --git a/src/StellaOps.Authority/StellaOps.Authority.Plugins.Abstractions/AuthorityClientMetadataKeys.cs b/src/StellaOps.Authority/StellaOps.Authority.Plugins.Abstractions/AuthorityClientMetadataKeys.cs new file mode 100644 index 00000000..96e4d631 --- /dev/null +++ b/src/StellaOps.Authority/StellaOps.Authority.Plugins.Abstractions/AuthorityClientMetadataKeys.cs @@ -0,0 +1,12 @@ +namespace StellaOps.Authority.Plugins.Abstractions; + +/// +/// Well-known metadata keys persisted with Authority client registrations. +/// +public static class AuthorityClientMetadataKeys +{ + public const string AllowedGrantTypes = "allowedGrantTypes"; + public const string AllowedScopes = "allowedScopes"; + public const string RedirectUris = "redirectUris"; + public const string PostLogoutRedirectUris = "postLogoutRedirectUris"; +} diff --git a/src/StellaOps.Authority/StellaOps.Authority.Plugins.Abstractions/AuthorityPluginContracts.cs b/src/StellaOps.Authority/StellaOps.Authority.Plugins.Abstractions/AuthorityPluginContracts.cs new file mode 100644 index 00000000..6eb184ee --- /dev/null +++ b/src/StellaOps.Authority/StellaOps.Authority.Plugins.Abstractions/AuthorityPluginContracts.cs @@ -0,0 +1,139 @@ +using System; +using System.Collections.Generic; +using System.Diagnostics.CodeAnalysis; +using Microsoft.Extensions.Configuration; + +namespace StellaOps.Authority.Plugins.Abstractions; + +/// +/// Well-known Authority plugin capability identifiers. +/// +public static class AuthorityPluginCapabilities +{ + public const string Password = "password"; + public const string Bootstrap = "bootstrap"; + public const string Mfa = "mfa"; + public const string ClientProvisioning = "clientProvisioning"; +} + +/// +/// Immutable description of an Authority plugin loaded from configuration. +/// +/// Logical name derived from configuration key. +/// Plugin type identifier (used for capability routing). +/// Whether the plugin is enabled. +/// Assembly name without extension. +/// Explicit assembly path override. +/// Capability hints exposed by the plugin. +/// Additional metadata forwarded to plugin implementations. +/// Absolute path to the plugin configuration manifest. +public sealed record AuthorityPluginManifest( + string Name, + string Type, + bool Enabled, + string? AssemblyName, + string? AssemblyPath, + IReadOnlyList Capabilities, + IReadOnlyDictionary Metadata, + string ConfigPath) +{ + /// + /// Determines whether the manifest declares the specified capability. + /// + /// Capability identifier to check. + public bool HasCapability(string capability) + { + if (string.IsNullOrWhiteSpace(capability)) + { + return false; + } + + foreach (var entry in Capabilities) + { + if (string.Equals(entry, capability, StringComparison.OrdinalIgnoreCase)) + { + return true; + } + } + + return false; + } +} + +/// +/// Runtime context combining plugin manifest metadata and its bound configuration. +/// +/// Manifest describing the plugin. +/// Root configuration built from the plugin YAML manifest. +public sealed record AuthorityPluginContext( + AuthorityPluginManifest Manifest, + IConfiguration Configuration); + +/// +/// Registry exposing the set of Authority plugins loaded at runtime. +/// +public interface IAuthorityPluginRegistry +{ + IReadOnlyCollection Plugins { get; } + + bool TryGet(string name, [NotNullWhen(true)] out AuthorityPluginContext? context); + + AuthorityPluginContext GetRequired(string name) + { + if (TryGet(name, out var context)) + { + return context; + } + + throw new KeyNotFoundException($"Authority plugin '{name}' is not registered."); + } +} + +/// +/// Registry exposing loaded identity provider plugins and their capabilities. +/// +public interface IAuthorityIdentityProviderRegistry +{ + /// + /// Gets all registered identity provider plugins keyed by logical name. + /// + IReadOnlyCollection Providers { get; } + + /// + /// Gets identity providers that advertise password support. + /// + IReadOnlyCollection PasswordProviders { get; } + + /// + /// Gets identity providers that advertise multi-factor authentication support. + /// + IReadOnlyCollection MfaProviders { get; } + + /// + /// Gets identity providers that advertise client provisioning support. + /// + IReadOnlyCollection ClientProvisioningProviders { get; } + + /// + /// Aggregate capability flags across all registered providers. + /// + AuthorityIdentityProviderCapabilities AggregateCapabilities { get; } + + /// + /// Attempts to resolve an identity provider by name. + /// + bool TryGet(string name, [NotNullWhen(true)] out IIdentityProviderPlugin? provider); + + /// + /// Resolves an identity provider by name or throws when not found. + /// + IIdentityProviderPlugin GetRequired(string name) + { + if (TryGet(name, out var provider)) + { + return provider; + } + + throw new KeyNotFoundException($"Identity provider plugin '{name}' is not registered."); + } +} diff --git a/src/StellaOps.Authority/StellaOps.Authority.Plugins.Abstractions/AuthorityPluginRegistrationContext.cs b/src/StellaOps.Authority/StellaOps.Authority.Plugins.Abstractions/AuthorityPluginRegistrationContext.cs new file mode 100644 index 00000000..509b8a96 --- /dev/null +++ b/src/StellaOps.Authority/StellaOps.Authority.Plugins.Abstractions/AuthorityPluginRegistrationContext.cs @@ -0,0 +1,60 @@ +using System; +using Microsoft.Extensions.Configuration; +using Microsoft.Extensions.DependencyInjection; + +namespace StellaOps.Authority.Plugins.Abstractions; + +/// +/// Provides shared services and metadata to Authority plugin registrars during DI setup. +/// +public sealed class AuthorityPluginRegistrationContext +{ + /// + /// Initialises a new registration context. + /// + /// Service collection used to register plugin services. + /// Plugin context describing the manifest and configuration. + /// Root host configuration available during registration. + /// Thrown when any argument is null. + public AuthorityPluginRegistrationContext( + IServiceCollection services, + AuthorityPluginContext plugin, + IConfiguration hostConfiguration) + { + Services = services ?? throw new ArgumentNullException(nameof(services)); + Plugin = plugin ?? throw new ArgumentNullException(nameof(plugin)); + HostConfiguration = hostConfiguration ?? throw new ArgumentNullException(nameof(hostConfiguration)); + } + + /// + /// Gets the service collection used to register plugin dependencies. + /// + public IServiceCollection Services { get; } + + /// + /// Gets the plugin context containing manifest metadata and configuration. + /// + public AuthorityPluginContext Plugin { get; } + + /// + /// Gets the root configuration associated with the Authority host. + /// + public IConfiguration HostConfiguration { get; } +} + +/// +/// Registers Authority plugin services for a specific plugin type. +/// +public interface IAuthorityPluginRegistrar +{ + /// + /// Logical plugin type identifier supported by this registrar (e.g. standard, ldap). + /// + string PluginType { get; } + + /// + /// Registers services for the supplied plugin context. + /// + /// Registration context containing services and metadata. + void Register(AuthorityPluginRegistrationContext context); +} diff --git a/src/StellaOps.Authority/StellaOps.Authority.Plugins.Abstractions/AuthoritySecretHasher.cs b/src/StellaOps.Authority/StellaOps.Authority.Plugins.Abstractions/AuthoritySecretHasher.cs new file mode 100644 index 00000000..82068db9 --- /dev/null +++ b/src/StellaOps.Authority/StellaOps.Authority.Plugins.Abstractions/AuthoritySecretHasher.cs @@ -0,0 +1,25 @@ +using System.Security.Cryptography; +using System.Text; + +namespace StellaOps.Authority.Plugins.Abstractions; + +/// +/// Deterministic hashing utilities for secrets managed by Authority plugins. +/// +public static class AuthoritySecretHasher +{ + /// + /// Computes a stable SHA-256 hash for the provided secret. + /// + public static string ComputeHash(string secret) + { + if (string.IsNullOrEmpty(secret)) + { + return string.Empty; + } + + using var sha256 = SHA256.Create(); + var bytes = sha256.ComputeHash(Encoding.UTF8.GetBytes(secret)); + return Convert.ToBase64String(bytes); + } +} diff --git a/src/StellaOps.Authority/StellaOps.Authority.Plugins.Abstractions/IdentityProviderContracts.cs b/src/StellaOps.Authority/StellaOps.Authority.Plugins.Abstractions/IdentityProviderContracts.cs new file mode 100644 index 00000000..53c4e79d --- /dev/null +++ b/src/StellaOps.Authority/StellaOps.Authority.Plugins.Abstractions/IdentityProviderContracts.cs @@ -0,0 +1,785 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Security.Claims; +using System.Threading; +using System.Threading.Tasks; + +namespace StellaOps.Authority.Plugins.Abstractions; + +/// +/// Describes feature support advertised by an identity provider plugin. +/// +public sealed record AuthorityIdentityProviderCapabilities( + bool SupportsPassword, + bool SupportsMfa, + bool SupportsClientProvisioning) +{ + /// + /// Builds capabilities metadata from a list of capability identifiers. + /// + public static AuthorityIdentityProviderCapabilities FromCapabilities(IEnumerable capabilities) + { + if (capabilities is null) + { + return new AuthorityIdentityProviderCapabilities(false, false, false); + } + + var seen = new HashSet(StringComparer.OrdinalIgnoreCase); + foreach (var entry in capabilities) + { + if (string.IsNullOrWhiteSpace(entry)) + { + continue; + } + + seen.Add(entry.Trim()); + } + + return new AuthorityIdentityProviderCapabilities( + SupportsPassword: seen.Contains(AuthorityPluginCapabilities.Password), + SupportsMfa: seen.Contains(AuthorityPluginCapabilities.Mfa), + SupportsClientProvisioning: seen.Contains(AuthorityPluginCapabilities.ClientProvisioning)); + } +} + +/// +/// Represents a loaded Authority identity provider plugin instance. +/// +public interface IIdentityProviderPlugin +{ + /// + /// Gets the logical name of the plugin instance (matches the manifest key). + /// + string Name { get; } + + /// + /// Gets the plugin type identifier (e.g. standard, ldap). + /// + string Type { get; } + + /// + /// Gets the plugin context comprising the manifest and bound configuration. + /// + AuthorityPluginContext Context { get; } + + /// + /// Gets the credential store responsible for authenticator validation and user provisioning. + /// + IUserCredentialStore Credentials { get; } + + /// + /// Gets the claims enricher applied to issued principals. + /// + IClaimsEnricher ClaimsEnricher { get; } + + /// + /// Gets the optional client provisioning store exposed by the plugin. + /// + IClientProvisioningStore? ClientProvisioning { get; } + + /// + /// Gets the capability metadata advertised by the plugin. + /// + AuthorityIdentityProviderCapabilities Capabilities { get; } + + /// + /// Evaluates the health of the plugin and backing data stores. + /// + /// Token used to cancel the operation. + /// Health result describing the plugin status. + ValueTask CheckHealthAsync(CancellationToken cancellationToken); +} + +/// +/// Supplies operations for validating credentials and managing user records. +/// +public interface IUserCredentialStore +{ + /// + /// Verifies the supplied username/password combination. + /// + ValueTask VerifyPasswordAsync( + string username, + string password, + CancellationToken cancellationToken); + + /// + /// Creates or updates a user record based on the supplied registration data. + /// + ValueTask> UpsertUserAsync( + AuthorityUserRegistration registration, + CancellationToken cancellationToken); + + /// + /// Attempts to resolve a user descriptor by its canonical subject identifier. + /// + ValueTask FindBySubjectAsync( + string subjectId, + CancellationToken cancellationToken); +} + +/// +/// Enriches issued principals with additional claims based on plugin-specific rules. +/// +public interface IClaimsEnricher +{ + /// + /// Adds or adjusts claims on the provided identity. + /// + ValueTask EnrichAsync( + ClaimsIdentity identity, + AuthorityClaimsEnrichmentContext context, + CancellationToken cancellationToken); +} + +/// +/// Manages client (machine-to-machine) provisioning for Authority. +/// +public interface IClientProvisioningStore +{ + /// + /// Creates or updates a client registration. + /// + ValueTask> CreateOrUpdateAsync( + AuthorityClientRegistration registration, + CancellationToken cancellationToken); + + /// + /// Attempts to resolve a client descriptor by its identifier. + /// + ValueTask FindByClientIdAsync( + string clientId, + CancellationToken cancellationToken); + + /// + /// Removes a client registration. + /// + ValueTask DeleteAsync( + string clientId, + CancellationToken cancellationToken); +} + +/// +/// Represents the health state of a plugin or backing store. +/// +public enum AuthorityPluginHealthStatus +{ + /// + /// Plugin is healthy and operational. + /// + Healthy, + + /// + /// Plugin is degraded but still usable (e.g. transient connectivity issues). + /// + Degraded, + + /// + /// Plugin is unavailable and cannot service requests. + /// + Unavailable +} + +/// +/// Result of a plugin health probe. +/// +public sealed record AuthorityPluginHealthResult +{ + private AuthorityPluginHealthResult( + AuthorityPluginHealthStatus status, + string? message, + IReadOnlyDictionary details) + { + Status = status; + Message = message; + Details = details; + } + + /// + /// Gets the overall status of the plugin. + /// + public AuthorityPluginHealthStatus Status { get; } + + /// + /// Gets an optional human-readable status description. + /// + public string? Message { get; } + + /// + /// Gets optional structured details for diagnostics. + /// + public IReadOnlyDictionary Details { get; } + + /// + /// Creates a healthy result. + /// + public static AuthorityPluginHealthResult Healthy( + string? message = null, + IReadOnlyDictionary? details = null) + => new(AuthorityPluginHealthStatus.Healthy, message, details ?? EmptyDetails); + + /// + /// Creates a degraded result. + /// + public static AuthorityPluginHealthResult Degraded( + string? message = null, + IReadOnlyDictionary? details = null) + => new(AuthorityPluginHealthStatus.Degraded, message, details ?? EmptyDetails); + + /// + /// Creates an unavailable result. + /// + public static AuthorityPluginHealthResult Unavailable( + string? message = null, + IReadOnlyDictionary? details = null) + => new(AuthorityPluginHealthStatus.Unavailable, message, details ?? EmptyDetails); + + private static readonly IReadOnlyDictionary EmptyDetails = + new Dictionary(StringComparer.OrdinalIgnoreCase); +} + +/// +/// Describes a canonical Authority user surfaced by a plugin. +/// +public sealed record AuthorityUserDescriptor +{ + /// + /// Initialises a new user descriptor. + /// + public AuthorityUserDescriptor( + string subjectId, + string username, + string? displayName, + bool requiresPasswordReset, + IReadOnlyCollection? roles = null, + IReadOnlyDictionary? attributes = null) + { + SubjectId = ValidateRequired(subjectId, nameof(subjectId)); + Username = ValidateRequired(username, nameof(username)); + DisplayName = displayName; + RequiresPasswordReset = requiresPasswordReset; + Roles = roles is null ? Array.Empty() : roles.ToArray(); + Attributes = attributes is null + ? new Dictionary(StringComparer.OrdinalIgnoreCase) + : new Dictionary(attributes, StringComparer.OrdinalIgnoreCase); + } + + /// + /// Stable subject identifier for token issuance. + /// + public string SubjectId { get; } + + /// + /// Canonical username (case-normalised). + /// + public string Username { get; } + + /// + /// Optional human-friendly display name. + /// + public string? DisplayName { get; } + + /// + /// Indicates whether the user must reset their password. + /// + public bool RequiresPasswordReset { get; } + + /// + /// Collection of role identifiers associated with the user. + /// + public IReadOnlyCollection Roles { get; } + + /// + /// Arbitrary plugin-defined attributes (used by claims enricher). + /// + public IReadOnlyDictionary Attributes { get; } + + private static string ValidateRequired(string value, string paramName) + => string.IsNullOrWhiteSpace(value) + ? throw new ArgumentException("Value cannot be null or whitespace.", paramName) + : value; +} + +/// +/// Outcome of a credential verification attempt. +/// +public sealed record AuthorityCredentialVerificationResult +{ + private AuthorityCredentialVerificationResult( + bool succeeded, + AuthorityUserDescriptor? user, + AuthorityCredentialFailureCode? failureCode, + string? message, + TimeSpan? retryAfter) + { + Succeeded = succeeded; + User = user; + FailureCode = failureCode; + Message = message; + RetryAfter = retryAfter; + } + + /// + /// Indicates whether the verification succeeded. + /// + public bool Succeeded { get; } + + /// + /// Resolved user descriptor when successful. + /// + public AuthorityUserDescriptor? User { get; } + + /// + /// Failure classification when unsuccessful. + /// + public AuthorityCredentialFailureCode? FailureCode { get; } + + /// + /// Optional message describing the outcome. + /// + public string? Message { get; } + + /// + /// Optional suggested retry interval (e.g. for lockouts). + /// + public TimeSpan? RetryAfter { get; } + + /// + /// Builds a successful verification result. + /// + public static AuthorityCredentialVerificationResult Success( + AuthorityUserDescriptor user, + string? message = null) + => new(true, user ?? throw new ArgumentNullException(nameof(user)), null, message, null); + + /// + /// Builds a failed verification result. + /// + public static AuthorityCredentialVerificationResult Failure( + AuthorityCredentialFailureCode failureCode, + string? message = null, + TimeSpan? retryAfter = null) + => new(false, null, failureCode, message, retryAfter); +} + +/// +/// Classifies credential verification failures. +/// +public enum AuthorityCredentialFailureCode +{ + /// + /// Username/password combination is invalid. + /// + InvalidCredentials, + + /// + /// Account is locked out (retry after a specified duration). + /// + LockedOut, + + /// + /// Password has expired and must be reset. + /// + PasswordExpired, + + /// + /// User must reset password before proceeding. + /// + RequiresPasswordReset, + + /// + /// Additional multi-factor authentication is required. + /// + RequiresMfa, + + /// + /// Unexpected failure occurred (see message for details). + /// + UnknownError +} + +/// +/// Represents a user provisioning request. +/// +public sealed record AuthorityUserRegistration +{ + /// + /// Initialises a new registration. + /// + public AuthorityUserRegistration( + string username, + string? password, + string? displayName, + string? email, + bool requirePasswordReset, + IReadOnlyCollection? roles = null, + IReadOnlyDictionary? attributes = null) + { + Username = ValidateRequired(username, nameof(username)); + Password = password; + DisplayName = displayName; + Email = email; + RequirePasswordReset = requirePasswordReset; + Roles = roles is null ? Array.Empty() : roles.ToArray(); + Attributes = attributes is null + ? new Dictionary(StringComparer.OrdinalIgnoreCase) + : new Dictionary(attributes, StringComparer.OrdinalIgnoreCase); + } + + /// + /// Canonical username (unique). + /// + public string Username { get; } + + /// + /// Optional raw password (hashed by plugin). + /// + public string? Password { get; init; } + + /// + /// Optional human-friendly display name. + /// + public string? DisplayName { get; } + + /// + /// Optional contact email. + /// + public string? Email { get; } + + /// + /// Indicates whether the user must reset their password at next login. + /// + public bool RequirePasswordReset { get; } + + /// + /// Associated roles. + /// + public IReadOnlyCollection Roles { get; } + + /// + /// Plugin-defined attributes. + /// + public IReadOnlyDictionary Attributes { get; } + + /// + /// Creates a copy with the provided password while preserving other fields. + /// + public AuthorityUserRegistration WithPassword(string? password) + => new(Username, password, DisplayName, Email, RequirePasswordReset, Roles, Attributes); + + private static string ValidateRequired(string value, string paramName) + => string.IsNullOrWhiteSpace(value) + ? throw new ArgumentException("Value cannot be null or whitespace.", paramName) + : value; +} + +/// +/// Generic operation result utilised by plugins. +/// +public sealed record AuthorityPluginOperationResult +{ + private AuthorityPluginOperationResult(bool succeeded, string? errorCode, string? message) + { + Succeeded = succeeded; + ErrorCode = errorCode; + Message = message; + } + + /// + /// Indicates whether the operation succeeded. + /// + public bool Succeeded { get; } + + /// + /// Machine-readable error code (populated on failure). + /// + public string? ErrorCode { get; } + + /// + /// Optional human-readable message. + /// + public string? Message { get; } + + /// + /// Returns a successful result. + /// + public static AuthorityPluginOperationResult Success(string? message = null) + => new(true, null, message); + + /// + /// Returns a failed result with the supplied error code. + /// + public static AuthorityPluginOperationResult Failure(string errorCode, string? message = null) + => new(false, ValidateErrorCode(errorCode), message); + + internal static string ValidateErrorCode(string errorCode) + => string.IsNullOrWhiteSpace(errorCode) + ? throw new ArgumentException("Error code is required for failures.", nameof(errorCode)) + : errorCode; +} + +/// +/// Generic operation result that returns a value. +/// +public sealed record AuthorityPluginOperationResult +{ + private AuthorityPluginOperationResult( + bool succeeded, + TValue? value, + string? errorCode, + string? message) + { + Succeeded = succeeded; + Value = value; + ErrorCode = errorCode; + Message = message; + } + + /// + /// Indicates whether the operation succeeded. + /// + public bool Succeeded { get; } + + /// + /// Returned value when successful. + /// + public TValue? Value { get; } + + /// + /// Machine-readable error code (on failure). + /// + public string? ErrorCode { get; } + + /// + /// Optional human-readable message. + /// + public string? Message { get; } + + /// + /// Returns a successful result with the provided value. + /// + public static AuthorityPluginOperationResult Success(TValue value, string? message = null) + => new(true, value, null, message); + + /// + /// Returns a successful result without a value (defaults to default). + /// + public static AuthorityPluginOperationResult Success(string? message = null) + => new(true, default, null, message); + + /// + /// Returns a failed result with the supplied error code. + /// + public static AuthorityPluginOperationResult Failure(string errorCode, string? message = null) + => new(false, default, AuthorityPluginOperationResult.ValidateErrorCode(errorCode), message); +} + +/// +/// Context supplied to claims enrichment routines. +/// +public sealed class AuthorityClaimsEnrichmentContext +{ + private readonly Dictionary items; + + /// + /// Initialises a new context instance. + /// + public AuthorityClaimsEnrichmentContext( + AuthorityPluginContext plugin, + AuthorityUserDescriptor? user, + AuthorityClientDescriptor? client) + { + Plugin = plugin ?? throw new ArgumentNullException(nameof(plugin)); + User = user; + Client = client; + items = new Dictionary(StringComparer.OrdinalIgnoreCase); + } + + /// + /// Gets the plugin context associated with the principal. + /// + public AuthorityPluginContext Plugin { get; } + + /// + /// Gets the user descriptor when available. + /// + public AuthorityUserDescriptor? User { get; } + + /// + /// Gets the client descriptor when available. + /// + public AuthorityClientDescriptor? Client { get; } + + /// + /// Extensible bag for plugin-specific data passed between enrichment stages. + /// + public IDictionary Items => items; +} + +/// +/// Represents a registered OAuth/OpenID client. +/// +public sealed record AuthorityClientDescriptor +{ + /// + /// Initialises a new client descriptor. + /// + public AuthorityClientDescriptor( + string clientId, + string? displayName, + bool confidential, + IReadOnlyCollection? allowedGrantTypes = null, + IReadOnlyCollection? allowedScopes = null, + IReadOnlyCollection? redirectUris = null, + IReadOnlyCollection? postLogoutRedirectUris = null, + IReadOnlyDictionary? properties = null) + { + ClientId = ValidateRequired(clientId, nameof(clientId)); + DisplayName = displayName; + Confidential = confidential; + AllowedGrantTypes = allowedGrantTypes is null ? Array.Empty() : allowedGrantTypes.ToArray(); + AllowedScopes = allowedScopes is null ? Array.Empty() : allowedScopes.ToArray(); + RedirectUris = redirectUris is null ? Array.Empty() : redirectUris.ToArray(); + PostLogoutRedirectUris = postLogoutRedirectUris is null ? Array.Empty() : postLogoutRedirectUris.ToArray(); + Properties = properties is null + ? new Dictionary(StringComparer.OrdinalIgnoreCase) + : new Dictionary(properties, StringComparer.OrdinalIgnoreCase); + } + + /// + /// Unique client identifier. + /// + public string ClientId { get; } + + /// + /// Optional display name. + /// + public string? DisplayName { get; } + + /// + /// Indicates whether the client is confidential (requires secret). + /// + public bool Confidential { get; } + + /// + /// Permitted OAuth grant types. + /// + public IReadOnlyCollection AllowedGrantTypes { get; } + + /// + /// Permitted scopes. + /// + public IReadOnlyCollection AllowedScopes { get; } + + /// + /// Registered redirect URIs. + /// + public IReadOnlyCollection RedirectUris { get; } + + /// + /// Registered post-logout redirect URIs. + /// + public IReadOnlyCollection PostLogoutRedirectUris { get; } + + /// + /// Additional plugin-defined metadata. + /// + public IReadOnlyDictionary Properties { get; } + + private static string ValidateRequired(string value, string paramName) + => string.IsNullOrWhiteSpace(value) + ? throw new ArgumentException("Value cannot be null or whitespace.", paramName) + : value; +} + +/// +/// Client registration payload used when provisioning clients through plugins. +/// +public sealed record AuthorityClientRegistration +{ + /// + /// Initialises a new registration. + /// + public AuthorityClientRegistration( + string clientId, + bool confidential, + string? displayName, + string? clientSecret, + IReadOnlyCollection? allowedGrantTypes = null, + IReadOnlyCollection? allowedScopes = null, + IReadOnlyCollection? redirectUris = null, + IReadOnlyCollection? postLogoutRedirectUris = null, + IReadOnlyDictionary? properties = null) + { + ClientId = ValidateRequired(clientId, nameof(clientId)); + Confidential = confidential; + DisplayName = displayName; + ClientSecret = confidential + ? ValidateRequired(clientSecret ?? string.Empty, nameof(clientSecret)) + : clientSecret; + AllowedGrantTypes = allowedGrantTypes is null ? Array.Empty() : allowedGrantTypes.ToArray(); + AllowedScopes = allowedScopes is null ? Array.Empty() : allowedScopes.ToArray(); + RedirectUris = redirectUris is null ? Array.Empty() : redirectUris.ToArray(); + PostLogoutRedirectUris = postLogoutRedirectUris is null ? Array.Empty() : postLogoutRedirectUris.ToArray(); + Properties = properties is null + ? new Dictionary(StringComparer.OrdinalIgnoreCase) + : new Dictionary(properties, StringComparer.OrdinalIgnoreCase); + } + + /// + /// Unique client identifier. + /// + public string ClientId { get; } + + /// + /// Indicates whether the client is confidential (requires secret handling). + /// + public bool Confidential { get; } + + /// + /// Optional display name. + /// + public string? DisplayName { get; } + + /// + /// Optional raw client secret (hashed by the plugin for storage). + /// + public string? ClientSecret { get; init; } + + /// + /// Grant types to enable. + /// + public IReadOnlyCollection AllowedGrantTypes { get; } + + /// + /// Scopes assigned to the client. + /// + public IReadOnlyCollection AllowedScopes { get; } + + /// + /// Redirect URIs permitted for the client. + /// + public IReadOnlyCollection RedirectUris { get; } + + /// + /// Post-logout redirect URIs. + /// + public IReadOnlyCollection PostLogoutRedirectUris { get; } + + /// + /// Additional metadata for the plugin. + /// + public IReadOnlyDictionary Properties { get; } + + /// + /// Creates a copy of the registration with the provided client secret. + /// + public AuthorityClientRegistration WithClientSecret(string? clientSecret) + => new(ClientId, Confidential, DisplayName, clientSecret, AllowedGrantTypes, AllowedScopes, RedirectUris, PostLogoutRedirectUris, Properties); + + private static string ValidateRequired(string value, string paramName) + => string.IsNullOrWhiteSpace(value) + ? throw new ArgumentException("Value cannot be null or whitespace.", paramName) + : value; +} diff --git a/src/StellaOps.Authority/StellaOps.Authority.Plugins.Abstractions/StellaOps.Authority.Plugins.Abstractions.csproj b/src/StellaOps.Authority/StellaOps.Authority.Plugins.Abstractions/StellaOps.Authority.Plugins.Abstractions.csproj new file mode 100644 index 00000000..6226aad6 --- /dev/null +++ b/src/StellaOps.Authority/StellaOps.Authority.Plugins.Abstractions/StellaOps.Authority.Plugins.Abstractions.csproj @@ -0,0 +1,14 @@ + + + net10.0 + preview + enable + enable + true + + + + + + + diff --git a/src/StellaOps.Authority/StellaOps.Authority.Storage.Mongo/AuthorityMongoDefaults.cs b/src/StellaOps.Authority/StellaOps.Authority.Storage.Mongo/AuthorityMongoDefaults.cs new file mode 100644 index 00000000..a5f25b25 --- /dev/null +++ b/src/StellaOps.Authority/StellaOps.Authority.Storage.Mongo/AuthorityMongoDefaults.cs @@ -0,0 +1,24 @@ +namespace StellaOps.Authority.Storage.Mongo; + +/// +/// Constants describing default collection names and other MongoDB defaults for the Authority service. +/// +public static class AuthorityMongoDefaults +{ + /// + /// Default database name used when none is provided via configuration. + /// + public const string DefaultDatabaseName = "stellaops_authority"; + + /// + /// Canonical collection names used by Authority. + /// + public static class Collections + { + public const string Users = "authority_users"; + public const string Clients = "authority_clients"; + public const string Scopes = "authority_scopes"; + public const string Tokens = "authority_tokens"; + public const string LoginAttempts = "authority_login_attempts"; + } +} diff --git a/src/StellaOps.Authority/StellaOps.Authority.Storage.Mongo/Class1.cs b/src/StellaOps.Authority/StellaOps.Authority.Storage.Mongo/Class1.cs new file mode 100644 index 00000000..ff679d64 --- /dev/null +++ b/src/StellaOps.Authority/StellaOps.Authority.Storage.Mongo/Class1.cs @@ -0,0 +1,6 @@ +namespace StellaOps.Authority.Storage.Mongo; + +public class Class1 +{ + +} diff --git a/src/StellaOps.Authority/StellaOps.Authority.Storage.Mongo/Documents/AuthorityClientDocument.cs b/src/StellaOps.Authority/StellaOps.Authority.Storage.Mongo/Documents/AuthorityClientDocument.cs new file mode 100644 index 00000000..42b8699b --- /dev/null +++ b/src/StellaOps.Authority/StellaOps.Authority.Storage.Mongo/Documents/AuthorityClientDocument.cs @@ -0,0 +1,61 @@ +using MongoDB.Bson; +using MongoDB.Bson.Serialization.Attributes; + +namespace StellaOps.Authority.Storage.Mongo.Documents; + +/// +/// Represents an OAuth client/application registered with Authority. +/// +[BsonIgnoreExtraElements] +public sealed class AuthorityClientDocument +{ + [BsonId] + [BsonRepresentation(BsonType.ObjectId)] + public string Id { get; set; } = ObjectId.GenerateNewId().ToString(); + + [BsonElement("clientId")] + public string ClientId { get; set; } = string.Empty; + + [BsonElement("clientType")] + public string ClientType { get; set; } = "confidential"; + + [BsonElement("displayName")] + [BsonIgnoreIfNull] + public string? DisplayName { get; set; } + + [BsonElement("description")] + [BsonIgnoreIfNull] + public string? Description { get; set; } + + [BsonElement("secretHash")] + [BsonIgnoreIfNull] + public string? SecretHash { get; set; } + + [BsonElement("permissions")] + public List Permissions { get; set; } = new(); + + [BsonElement("requirements")] + public List Requirements { get; set; } = new(); + + [BsonElement("redirectUris")] + public List RedirectUris { get; set; } = new(); + + [BsonElement("postLogoutRedirectUris")] + public List PostLogoutRedirectUris { get; set; } = new(); + + [BsonElement("properties")] + public Dictionary Properties { get; set; } = new(StringComparer.OrdinalIgnoreCase); + + [BsonElement("plugin")] + [BsonIgnoreIfNull] + public string? Plugin { get; set; } + + [BsonElement("createdAt")] + public DateTimeOffset CreatedAt { get; set; } = DateTimeOffset.UtcNow; + + [BsonElement("updatedAt")] + public DateTimeOffset UpdatedAt { get; set; } = DateTimeOffset.UtcNow; + + [BsonElement("disabled")] + public bool Disabled { get; set; } +} diff --git a/src/StellaOps.Authority/StellaOps.Authority.Storage.Mongo/Documents/AuthorityLoginAttemptDocument.cs b/src/StellaOps.Authority/StellaOps.Authority.Storage.Mongo/Documents/AuthorityLoginAttemptDocument.cs new file mode 100644 index 00000000..7006f3fb --- /dev/null +++ b/src/StellaOps.Authority/StellaOps.Authority.Storage.Mongo/Documents/AuthorityLoginAttemptDocument.cs @@ -0,0 +1,45 @@ +using MongoDB.Bson; +using MongoDB.Bson.Serialization.Attributes; + +namespace StellaOps.Authority.Storage.Mongo.Documents; + +/// +/// Represents a recorded login attempt for audit and lockout purposes. +/// +[BsonIgnoreExtraElements] +public sealed class AuthorityLoginAttemptDocument +{ + [BsonId] + [BsonRepresentation(BsonType.ObjectId)] + public string Id { get; set; } = ObjectId.GenerateNewId().ToString(); + + [BsonElement("subjectId")] + [BsonIgnoreIfNull] + public string? SubjectId { get; set; } + + [BsonElement("username")] + [BsonIgnoreIfNull] + public string? Username { get; set; } + + [BsonElement("clientId")] + [BsonIgnoreIfNull] + public string? ClientId { get; set; } + + [BsonElement("plugin")] + [BsonIgnoreIfNull] + public string? Plugin { get; set; } + + [BsonElement("successful")] + public bool Successful { get; set; } + + [BsonElement("reason")] + [BsonIgnoreIfNull] + public string? Reason { get; set; } + + [BsonElement("remoteAddress")] + [BsonIgnoreIfNull] + public string? RemoteAddress { get; set; } + + [BsonElement("occurredAt")] + public DateTimeOffset OccurredAt { get; set; } = DateTimeOffset.UtcNow; +} diff --git a/src/StellaOps.Authority/StellaOps.Authority.Storage.Mongo/Documents/AuthorityScopeDocument.cs b/src/StellaOps.Authority/StellaOps.Authority.Storage.Mongo/Documents/AuthorityScopeDocument.cs new file mode 100644 index 00000000..76379a4b --- /dev/null +++ b/src/StellaOps.Authority/StellaOps.Authority.Storage.Mongo/Documents/AuthorityScopeDocument.cs @@ -0,0 +1,38 @@ +using MongoDB.Bson; +using MongoDB.Bson.Serialization.Attributes; + +namespace StellaOps.Authority.Storage.Mongo.Documents; + +/// +/// Represents an OAuth scope exposed by Authority. +/// +[BsonIgnoreExtraElements] +public sealed class AuthorityScopeDocument +{ + [BsonId] + [BsonRepresentation(BsonType.ObjectId)] + public string Id { get; set; } = ObjectId.GenerateNewId().ToString(); + + [BsonElement("name")] + public string Name { get; set; } = string.Empty; + + [BsonElement("displayName")] + [BsonIgnoreIfNull] + public string? DisplayName { get; set; } + + [BsonElement("description")] + [BsonIgnoreIfNull] + public string? Description { get; set; } + + [BsonElement("resources")] + public List Resources { get; set; } = new(); + + [BsonElement("properties")] + public Dictionary Properties { get; set; } = new(StringComparer.OrdinalIgnoreCase); + + [BsonElement("createdAt")] + public DateTimeOffset CreatedAt { get; set; } = DateTimeOffset.UtcNow; + + [BsonElement("updatedAt")] + public DateTimeOffset UpdatedAt { get; set; } = DateTimeOffset.UtcNow; +} diff --git a/src/StellaOps.Authority/StellaOps.Authority.Storage.Mongo/Documents/AuthorityTokenDocument.cs b/src/StellaOps.Authority/StellaOps.Authority.Storage.Mongo/Documents/AuthorityTokenDocument.cs new file mode 100644 index 00000000..2f1d40f5 --- /dev/null +++ b/src/StellaOps.Authority/StellaOps.Authority.Storage.Mongo/Documents/AuthorityTokenDocument.cs @@ -0,0 +1,54 @@ +using MongoDB.Bson; +using MongoDB.Bson.Serialization.Attributes; + +namespace StellaOps.Authority.Storage.Mongo.Documents; + +/// +/// Represents an OAuth token issued by Authority. +/// +[BsonIgnoreExtraElements] +public sealed class AuthorityTokenDocument +{ + [BsonId] + [BsonRepresentation(BsonType.ObjectId)] + public string Id { get; set; } = ObjectId.GenerateNewId().ToString(); + + [BsonElement("tokenId")] + public string TokenId { get; set; } = Guid.NewGuid().ToString("N"); + + [BsonElement("type")] + public string Type { get; set; } = string.Empty; + + [BsonElement("subjectId")] + [BsonIgnoreIfNull] + public string? SubjectId { get; set; } + + [BsonElement("clientId")] + [BsonIgnoreIfNull] + public string? ClientId { get; set; } + + [BsonElement("scope")] + public List Scope { get; set; } = new(); + + [BsonElement("referenceId")] + [BsonIgnoreIfNull] + public string? ReferenceId { get; set; } + + [BsonElement("status")] + public string Status { get; set; } = "valid"; + + [BsonElement("payload")] + [BsonIgnoreIfNull] + public string? Payload { get; set; } + + [BsonElement("createdAt")] + public DateTimeOffset CreatedAt { get; set; } = DateTimeOffset.UtcNow; + + [BsonElement("expiresAt")] + [BsonIgnoreIfNull] + public DateTimeOffset? ExpiresAt { get; set; } + + [BsonElement("revokedAt")] + [BsonIgnoreIfNull] + public DateTimeOffset? RevokedAt { get; set; } +} diff --git a/src/StellaOps.Authority/StellaOps.Authority.Storage.Mongo/Documents/AuthorityUserDocument.cs b/src/StellaOps.Authority/StellaOps.Authority.Storage.Mongo/Documents/AuthorityUserDocument.cs new file mode 100644 index 00000000..09e7a3c4 --- /dev/null +++ b/src/StellaOps.Authority/StellaOps.Authority.Storage.Mongo/Documents/AuthorityUserDocument.cs @@ -0,0 +1,51 @@ +using MongoDB.Bson; +using MongoDB.Bson.Serialization.Attributes; + +namespace StellaOps.Authority.Storage.Mongo.Documents; + +/// +/// Represents a canonical Authority user persisted in MongoDB. +/// +[BsonIgnoreExtraElements] +public sealed class AuthorityUserDocument +{ + [BsonId] + [BsonRepresentation(BsonType.ObjectId)] + public string Id { get; set; } = ObjectId.GenerateNewId().ToString(); + + [BsonElement("subjectId")] + public string SubjectId { get; set; } = Guid.NewGuid().ToString("N"); + + [BsonElement("username")] + public string Username { get; set; } = string.Empty; + + [BsonElement("normalizedUsername")] + public string NormalizedUsername { get; set; } = string.Empty; + + [BsonElement("displayName")] + [BsonIgnoreIfNull] + public string? DisplayName { get; set; } + + [BsonElement("email")] + [BsonIgnoreIfNull] + public string? Email { get; set; } + + [BsonElement("disabled")] + public bool Disabled { get; set; } + + [BsonElement("roles")] + public List Roles { get; set; } = new(); + + [BsonElement("attributes")] + public Dictionary Attributes { get; set; } = new(StringComparer.OrdinalIgnoreCase); + + [BsonElement("plugin")] + [BsonIgnoreIfNull] + public string? Plugin { get; set; } + + [BsonElement("createdAt")] + public DateTimeOffset CreatedAt { get; set; } = DateTimeOffset.UtcNow; + + [BsonElement("updatedAt")] + public DateTimeOffset UpdatedAt { get; set; } = DateTimeOffset.UtcNow; +} diff --git a/src/StellaOps.Authority/StellaOps.Authority.Storage.Mongo/Extensions/ServiceCollectionExtensions.cs b/src/StellaOps.Authority/StellaOps.Authority.Storage.Mongo/Extensions/ServiceCollectionExtensions.cs new file mode 100644 index 00000000..8722c8e7 --- /dev/null +++ b/src/StellaOps.Authority/StellaOps.Authority.Storage.Mongo/Extensions/ServiceCollectionExtensions.cs @@ -0,0 +1,103 @@ +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.DependencyInjection.Extensions; +using Microsoft.Extensions.Options; +using MongoDB.Driver; +using StellaOps.Authority.Storage.Mongo.Documents; +using StellaOps.Authority.Storage.Mongo.Initialization; +using StellaOps.Authority.Storage.Mongo.Migrations; +using StellaOps.Authority.Storage.Mongo.Options; +using StellaOps.Authority.Storage.Mongo.Stores; + +namespace StellaOps.Authority.Storage.Mongo.Extensions; + +/// +/// Dependency injection helpers for wiring the Authority MongoDB storage layer. +/// +public static class ServiceCollectionExtensions +{ + public static IServiceCollection AddAuthorityMongoStorage( + this IServiceCollection services, + Action configureOptions) + { + ArgumentNullException.ThrowIfNull(services); + ArgumentNullException.ThrowIfNull(configureOptions); + + services.AddOptions() + .Configure(configureOptions) + .PostConfigure(static options => options.EnsureValid()); + + services.TryAddSingleton(TimeProvider.System); + + services.AddSingleton(static sp => + { + var options = sp.GetRequiredService>().Value; + return new MongoClient(options.ConnectionString); + }); + + services.AddSingleton(static sp => + { + var options = sp.GetRequiredService>().Value; + var client = sp.GetRequiredService(); + + var settings = new MongoDatabaseSettings + { + ReadConcern = ReadConcern.Majority, + WriteConcern = WriteConcern.WMajority, + ReadPreference = ReadPreference.PrimaryPreferred + }; + + var database = client.GetDatabase(options.GetDatabaseName(), settings); + var writeConcern = database.Settings.WriteConcern.With(wTimeout: options.CommandTimeout); + return database.WithWriteConcern(writeConcern); + }); + + services.AddSingleton(); + services.AddSingleton(); + + services.TryAddEnumerable(ServiceDescriptor.Singleton()); + + services.AddSingleton(static sp => + { + var database = sp.GetRequiredService(); + return database.GetCollection(AuthorityMongoDefaults.Collections.Users); + }); + + services.AddSingleton(static sp => + { + var database = sp.GetRequiredService(); + return database.GetCollection(AuthorityMongoDefaults.Collections.Clients); + }); + + services.AddSingleton(static sp => + { + var database = sp.GetRequiredService(); + return database.GetCollection(AuthorityMongoDefaults.Collections.Scopes); + }); + + services.AddSingleton(static sp => + { + var database = sp.GetRequiredService(); + return database.GetCollection(AuthorityMongoDefaults.Collections.Tokens); + }); + + services.AddSingleton(static sp => + { + var database = sp.GetRequiredService(); + return database.GetCollection(AuthorityMongoDefaults.Collections.LoginAttempts); + }); + + services.TryAddSingleton(); + services.TryAddSingleton(); + services.TryAddSingleton(); + services.TryAddSingleton(); + services.TryAddSingleton(); + + services.TryAddSingleton(); + services.TryAddSingleton(); + services.TryAddSingleton(); + services.TryAddSingleton(); + services.TryAddSingleton(); + + return services; + } +} diff --git a/src/StellaOps.Authority/StellaOps.Authority.Storage.Mongo/Initialization/AuthorityClientCollectionInitializer.cs b/src/StellaOps.Authority/StellaOps.Authority.Storage.Mongo/Initialization/AuthorityClientCollectionInitializer.cs new file mode 100644 index 00000000..2f21c239 --- /dev/null +++ b/src/StellaOps.Authority/StellaOps.Authority.Storage.Mongo/Initialization/AuthorityClientCollectionInitializer.cs @@ -0,0 +1,24 @@ +using MongoDB.Driver; +using StellaOps.Authority.Storage.Mongo.Documents; + +namespace StellaOps.Authority.Storage.Mongo.Initialization; + +internal sealed class AuthorityClientCollectionInitializer : IAuthorityCollectionInitializer +{ + public async ValueTask EnsureIndexesAsync(IMongoDatabase database, CancellationToken cancellationToken) + { + var collection = database.GetCollection(AuthorityMongoDefaults.Collections.Clients); + + var indexModels = new[] + { + new CreateIndexModel( + Builders.IndexKeys.Ascending(c => c.ClientId), + new CreateIndexOptions { Name = "client_id_unique", Unique = true }), + new CreateIndexModel( + Builders.IndexKeys.Ascending(c => c.Disabled), + new CreateIndexOptions { Name = "client_disabled" }) + }; + + await collection.Indexes.CreateManyAsync(indexModels, cancellationToken).ConfigureAwait(false); + } +} diff --git a/src/StellaOps.Authority/StellaOps.Authority.Storage.Mongo/Initialization/AuthorityLoginAttemptCollectionInitializer.cs b/src/StellaOps.Authority/StellaOps.Authority.Storage.Mongo/Initialization/AuthorityLoginAttemptCollectionInitializer.cs new file mode 100644 index 00000000..b1b85bdd --- /dev/null +++ b/src/StellaOps.Authority/StellaOps.Authority.Storage.Mongo/Initialization/AuthorityLoginAttemptCollectionInitializer.cs @@ -0,0 +1,26 @@ +using MongoDB.Driver; +using StellaOps.Authority.Storage.Mongo.Documents; + +namespace StellaOps.Authority.Storage.Mongo.Initialization; + +internal sealed class AuthorityLoginAttemptCollectionInitializer : IAuthorityCollectionInitializer +{ + public async ValueTask EnsureIndexesAsync(IMongoDatabase database, CancellationToken cancellationToken) + { + var collection = database.GetCollection(AuthorityMongoDefaults.Collections.LoginAttempts); + + var indexModels = new[] + { + new CreateIndexModel( + Builders.IndexKeys + .Ascending(a => a.SubjectId) + .Descending(a => a.OccurredAt), + new CreateIndexOptions { Name = "login_attempt_subject_time" }), + new CreateIndexModel( + Builders.IndexKeys.Descending(a => a.OccurredAt), + new CreateIndexOptions { Name = "login_attempt_time" }) + }; + + await collection.Indexes.CreateManyAsync(indexModels, cancellationToken).ConfigureAwait(false); + } +} diff --git a/src/StellaOps.Authority/StellaOps.Authority.Storage.Mongo/Initialization/AuthorityMongoInitializer.cs b/src/StellaOps.Authority/StellaOps.Authority.Storage.Mongo/Initialization/AuthorityMongoInitializer.cs new file mode 100644 index 00000000..b4a0332c --- /dev/null +++ b/src/StellaOps.Authority/StellaOps.Authority.Storage.Mongo/Initialization/AuthorityMongoInitializer.cs @@ -0,0 +1,55 @@ +using Microsoft.Extensions.Logging; +using MongoDB.Driver; +using StellaOps.Authority.Storage.Mongo.Migrations; + +namespace StellaOps.Authority.Storage.Mongo.Initialization; + +/// +/// Performs MongoDB bootstrap tasks for the Authority service. +/// +public sealed class AuthorityMongoInitializer +{ + private readonly IEnumerable collectionInitializers; + private readonly AuthorityMongoMigrationRunner migrationRunner; + private readonly ILogger logger; + + public AuthorityMongoInitializer( + IEnumerable collectionInitializers, + AuthorityMongoMigrationRunner migrationRunner, + ILogger logger) + { + this.collectionInitializers = collectionInitializers ?? throw new ArgumentNullException(nameof(collectionInitializers)); + this.migrationRunner = migrationRunner ?? throw new ArgumentNullException(nameof(migrationRunner)); + this.logger = logger ?? throw new ArgumentNullException(nameof(logger)); + } + + /// + /// Ensures collections exist, migrations run, and indexes are applied. + /// + public async ValueTask InitialiseAsync(IMongoDatabase database, CancellationToken cancellationToken) + { + ArgumentNullException.ThrowIfNull(database); + + await migrationRunner.RunAsync(database, cancellationToken).ConfigureAwait(false); + + foreach (var initializer in collectionInitializers) + { + try + { + logger.LogInformation( + "Ensuring Authority Mongo indexes via {InitializerType}.", + initializer.GetType().FullName); + + await initializer.EnsureIndexesAsync(database, cancellationToken).ConfigureAwait(false); + } + catch (Exception ex) + { + logger.LogError( + ex, + "Authority Mongo index initialisation failed for {InitializerType}.", + initializer.GetType().FullName); + throw; + } + } + } +} diff --git a/src/StellaOps.Authority/StellaOps.Authority.Storage.Mongo/Initialization/AuthorityScopeCollectionInitializer.cs b/src/StellaOps.Authority/StellaOps.Authority.Storage.Mongo/Initialization/AuthorityScopeCollectionInitializer.cs new file mode 100644 index 00000000..b744ee97 --- /dev/null +++ b/src/StellaOps.Authority/StellaOps.Authority.Storage.Mongo/Initialization/AuthorityScopeCollectionInitializer.cs @@ -0,0 +1,21 @@ +using MongoDB.Driver; +using StellaOps.Authority.Storage.Mongo.Documents; + +namespace StellaOps.Authority.Storage.Mongo.Initialization; + +internal sealed class AuthorityScopeCollectionInitializer : IAuthorityCollectionInitializer +{ + public async ValueTask EnsureIndexesAsync(IMongoDatabase database, CancellationToken cancellationToken) + { + var collection = database.GetCollection(AuthorityMongoDefaults.Collections.Scopes); + + var indexModels = new[] + { + new CreateIndexModel( + Builders.IndexKeys.Ascending(s => s.Name), + new CreateIndexOptions { Name = "scope_name_unique", Unique = true }) + }; + + await collection.Indexes.CreateManyAsync(indexModels, cancellationToken).ConfigureAwait(false); + } +} diff --git a/src/StellaOps.Authority/StellaOps.Authority.Storage.Mongo/Initialization/AuthorityTokenCollectionInitializer.cs b/src/StellaOps.Authority/StellaOps.Authority.Storage.Mongo/Initialization/AuthorityTokenCollectionInitializer.cs new file mode 100644 index 00000000..725e9762 --- /dev/null +++ b/src/StellaOps.Authority/StellaOps.Authority.Storage.Mongo/Initialization/AuthorityTokenCollectionInitializer.cs @@ -0,0 +1,40 @@ +using MongoDB.Driver; +using StellaOps.Authority.Storage.Mongo.Documents; + +namespace StellaOps.Authority.Storage.Mongo.Initialization; + +internal sealed class AuthorityTokenCollectionInitializer : IAuthorityCollectionInitializer +{ + public async ValueTask EnsureIndexesAsync(IMongoDatabase database, CancellationToken cancellationToken) + { + var collection = database.GetCollection(AuthorityMongoDefaults.Collections.Tokens); + + var indexModels = new List> + { + new( + Builders.IndexKeys.Ascending(t => t.TokenId), + new CreateIndexOptions { Name = "token_id_unique", Unique = true }), + new( + Builders.IndexKeys.Ascending(t => t.ReferenceId), + new CreateIndexOptions { Name = "token_reference_unique", Unique = true, Sparse = true }), + new( + Builders.IndexKeys.Ascending(t => t.SubjectId), + new CreateIndexOptions { Name = "token_subject" }), + new( + Builders.IndexKeys.Ascending(t => t.ClientId), + new CreateIndexOptions { Name = "token_client" }) + }; + + var expirationFilter = Builders.Filter.Exists(t => t.ExpiresAt, true); + indexModels.Add(new CreateIndexModel( + Builders.IndexKeys.Ascending(t => t.ExpiresAt), + new CreateIndexOptions + { + Name = "token_expiry_ttl", + ExpireAfter = TimeSpan.Zero, + PartialFilterExpression = expirationFilter + })); + + await collection.Indexes.CreateManyAsync(indexModels, cancellationToken).ConfigureAwait(false); + } +} diff --git a/src/StellaOps.Authority/StellaOps.Authority.Storage.Mongo/Initialization/AuthorityUserCollectionInitializer.cs b/src/StellaOps.Authority/StellaOps.Authority.Storage.Mongo/Initialization/AuthorityUserCollectionInitializer.cs new file mode 100644 index 00000000..61a682f9 --- /dev/null +++ b/src/StellaOps.Authority/StellaOps.Authority.Storage.Mongo/Initialization/AuthorityUserCollectionInitializer.cs @@ -0,0 +1,27 @@ +using MongoDB.Driver; +using StellaOps.Authority.Storage.Mongo.Documents; + +namespace StellaOps.Authority.Storage.Mongo.Initialization; + +internal sealed class AuthorityUserCollectionInitializer : IAuthorityCollectionInitializer +{ + public async ValueTask EnsureIndexesAsync(IMongoDatabase database, CancellationToken cancellationToken) + { + var collection = database.GetCollection(AuthorityMongoDefaults.Collections.Users); + + var indexModels = new[] + { + new CreateIndexModel( + Builders.IndexKeys.Ascending(u => u.SubjectId), + new CreateIndexOptions { Name = "user_subject_unique", Unique = true }), + new CreateIndexModel( + Builders.IndexKeys.Ascending(u => u.NormalizedUsername), + new CreateIndexOptions { Name = "user_normalized_username_unique", Unique = true, Sparse = true }), + new CreateIndexModel( + Builders.IndexKeys.Ascending(u => u.Email), + new CreateIndexOptions { Name = "user_email", Sparse = true }) + }; + + await collection.Indexes.CreateManyAsync(indexModels, cancellationToken).ConfigureAwait(false); + } +} diff --git a/src/StellaOps.Authority/StellaOps.Authority.Storage.Mongo/Initialization/IAuthorityCollectionInitializer.cs b/src/StellaOps.Authority/StellaOps.Authority.Storage.Mongo/Initialization/IAuthorityCollectionInitializer.cs new file mode 100644 index 00000000..2498427f --- /dev/null +++ b/src/StellaOps.Authority/StellaOps.Authority.Storage.Mongo/Initialization/IAuthorityCollectionInitializer.cs @@ -0,0 +1,14 @@ +using MongoDB.Driver; + +namespace StellaOps.Authority.Storage.Mongo.Initialization; + +/// +/// Persists indexes and configuration for an Authority Mongo collection. +/// +public interface IAuthorityCollectionInitializer +{ + /// + /// Ensures the collection's indexes exist. + /// + ValueTask EnsureIndexesAsync(IMongoDatabase database, CancellationToken cancellationToken); +} diff --git a/src/StellaOps.Authority/StellaOps.Authority.Storage.Mongo/Migrations/AuthorityMongoMigrationRunner.cs b/src/StellaOps.Authority/StellaOps.Authority.Storage.Mongo/Migrations/AuthorityMongoMigrationRunner.cs new file mode 100644 index 00000000..ce4e6080 --- /dev/null +++ b/src/StellaOps.Authority/StellaOps.Authority.Storage.Mongo/Migrations/AuthorityMongoMigrationRunner.cs @@ -0,0 +1,40 @@ +using Microsoft.Extensions.Logging; +using MongoDB.Driver; + +namespace StellaOps.Authority.Storage.Mongo.Migrations; + +/// +/// Executes registered Authority Mongo migrations sequentially. +/// +public sealed class AuthorityMongoMigrationRunner +{ + private readonly IEnumerable migrations; + private readonly ILogger logger; + + public AuthorityMongoMigrationRunner( + IEnumerable migrations, + ILogger logger) + { + this.migrations = migrations ?? throw new ArgumentNullException(nameof(migrations)); + this.logger = logger ?? throw new ArgumentNullException(nameof(logger)); + } + + public async ValueTask RunAsync(IMongoDatabase database, CancellationToken cancellationToken) + { + ArgumentNullException.ThrowIfNull(database); + + foreach (var migration in migrations) + { + try + { + logger.LogInformation("Running Authority Mongo migration {MigrationType}.", migration.GetType().FullName); + await migration.ExecuteAsync(database, cancellationToken).ConfigureAwait(false); + } + catch (Exception ex) + { + logger.LogError(ex, "Authority Mongo migration {MigrationType} failed.", migration.GetType().FullName); + throw; + } + } + } +} diff --git a/src/StellaOps.Authority/StellaOps.Authority.Storage.Mongo/Migrations/EnsureAuthorityCollectionsMigration.cs b/src/StellaOps.Authority/StellaOps.Authority.Storage.Mongo/Migrations/EnsureAuthorityCollectionsMigration.cs new file mode 100644 index 00000000..295c7f02 --- /dev/null +++ b/src/StellaOps.Authority/StellaOps.Authority.Storage.Mongo/Migrations/EnsureAuthorityCollectionsMigration.cs @@ -0,0 +1,44 @@ +using Microsoft.Extensions.Logging; +using MongoDB.Bson; +using MongoDB.Driver; + +namespace StellaOps.Authority.Storage.Mongo.Migrations; + +/// +/// Ensures base Authority collections exist prior to applying indexes. +/// +internal sealed class EnsureAuthorityCollectionsMigration : IAuthorityMongoMigration +{ + private static readonly string[] RequiredCollections = + { + AuthorityMongoDefaults.Collections.Users, + AuthorityMongoDefaults.Collections.Clients, + AuthorityMongoDefaults.Collections.Scopes, + AuthorityMongoDefaults.Collections.Tokens, + AuthorityMongoDefaults.Collections.LoginAttempts + }; + + private readonly ILogger logger; + + public EnsureAuthorityCollectionsMigration(ILogger logger) + => this.logger = logger ?? throw new ArgumentNullException(nameof(logger)); + + public async ValueTask ExecuteAsync(IMongoDatabase database, CancellationToken cancellationToken) + { + ArgumentNullException.ThrowIfNull(database); + + var existing = await database.ListCollectionNamesAsync(cancellationToken: cancellationToken).ConfigureAwait(false); + var existingNames = await existing.ToListAsync(cancellationToken).ConfigureAwait(false); + + foreach (var collection in RequiredCollections) + { + if (existingNames.Contains(collection, StringComparer.OrdinalIgnoreCase)) + { + continue; + } + + logger.LogInformation("Creating Authority Mongo collection '{CollectionName}'.", collection); + await database.CreateCollectionAsync(collection, cancellationToken: cancellationToken).ConfigureAwait(false); + } + } +} diff --git a/src/StellaOps.Authority/StellaOps.Authority.Storage.Mongo/Migrations/IAuthorityMongoMigration.cs b/src/StellaOps.Authority/StellaOps.Authority.Storage.Mongo/Migrations/IAuthorityMongoMigration.cs new file mode 100644 index 00000000..e6fb9d25 --- /dev/null +++ b/src/StellaOps.Authority/StellaOps.Authority.Storage.Mongo/Migrations/IAuthorityMongoMigration.cs @@ -0,0 +1,16 @@ +using MongoDB.Driver; + +namespace StellaOps.Authority.Storage.Mongo.Migrations; + +/// +/// Represents a Mongo migration run during Authority bootstrap. +/// +public interface IAuthorityMongoMigration +{ + /// + /// Executes the migration. + /// + /// Mongo database instance. + /// Cancellation token. + ValueTask ExecuteAsync(IMongoDatabase database, CancellationToken cancellationToken); +} diff --git a/src/StellaOps.Authority/StellaOps.Authority.Storage.Mongo/Options/AuthorityMongoOptions.cs b/src/StellaOps.Authority/StellaOps.Authority.Storage.Mongo/Options/AuthorityMongoOptions.cs new file mode 100644 index 00000000..8cc8f566 --- /dev/null +++ b/src/StellaOps.Authority/StellaOps.Authority.Storage.Mongo/Options/AuthorityMongoOptions.cs @@ -0,0 +1,64 @@ +using MongoDB.Driver; + +namespace StellaOps.Authority.Storage.Mongo.Options; + +/// +/// Strongly typed configuration for the StellaOps Authority MongoDB storage layer. +/// +public sealed class AuthorityMongoOptions +{ + /// + /// MongoDB connection string used to bootstrap the client. + /// + public string ConnectionString { get; set; } = string.Empty; + + /// + /// Optional override for the database name. When omitted the database name embedded in the connection string is used. + /// + public string? DatabaseName { get; set; } + + /// + /// Command timeout applied to MongoDB operations. + /// + public TimeSpan CommandTimeout { get; set; } = TimeSpan.FromSeconds(30); + + /// + /// Returns the resolved database name. + /// + public string GetDatabaseName() + { + if (!string.IsNullOrWhiteSpace(DatabaseName)) + { + return DatabaseName.Trim(); + } + + if (!string.IsNullOrWhiteSpace(ConnectionString)) + { + var url = MongoUrl.Create(ConnectionString); + if (!string.IsNullOrWhiteSpace(url.DatabaseName)) + { + return url.DatabaseName; + } + } + + return AuthorityMongoDefaults.DefaultDatabaseName; + } + + /// + /// Validates configured values and throws when invalid. + /// + public void EnsureValid() + { + if (string.IsNullOrWhiteSpace(ConnectionString)) + { + throw new InvalidOperationException("Authority Mongo storage requires a connection string."); + } + + if (CommandTimeout <= TimeSpan.Zero) + { + throw new InvalidOperationException("Authority Mongo storage command timeout must be greater than zero."); + } + + _ = GetDatabaseName(); + } +} diff --git a/src/StellaOps.Authority/StellaOps.Authority.Storage.Mongo/StellaOps.Authority.Storage.Mongo.csproj b/src/StellaOps.Authority/StellaOps.Authority.Storage.Mongo/StellaOps.Authority.Storage.Mongo.csproj new file mode 100644 index 00000000..44d7ac06 --- /dev/null +++ b/src/StellaOps.Authority/StellaOps.Authority.Storage.Mongo/StellaOps.Authority.Storage.Mongo.csproj @@ -0,0 +1,18 @@ + + + net10.0 + preview + enable + enable + true + + + + + + + + + + + diff --git a/src/StellaOps.Authority/StellaOps.Authority.Storage.Mongo/Stores/AuthorityClientStore.cs b/src/StellaOps.Authority/StellaOps.Authority.Storage.Mongo/Stores/AuthorityClientStore.cs new file mode 100644 index 00000000..b7f0bc2e --- /dev/null +++ b/src/StellaOps.Authority/StellaOps.Authority.Storage.Mongo/Stores/AuthorityClientStore.cs @@ -0,0 +1,64 @@ +using Microsoft.Extensions.Logging; +using MongoDB.Driver; +using StellaOps.Authority.Storage.Mongo.Documents; + +namespace StellaOps.Authority.Storage.Mongo.Stores; + +internal sealed class AuthorityClientStore : IAuthorityClientStore +{ + private readonly IMongoCollection collection; + private readonly TimeProvider clock; + private readonly ILogger logger; + + public AuthorityClientStore( + IMongoCollection collection, + TimeProvider clock, + ILogger logger) + { + this.collection = collection ?? throw new ArgumentNullException(nameof(collection)); + this.clock = clock ?? throw new ArgumentNullException(nameof(clock)); + this.logger = logger ?? throw new ArgumentNullException(nameof(logger)); + } + + public async ValueTask FindByClientIdAsync(string clientId, CancellationToken cancellationToken) + { + if (string.IsNullOrWhiteSpace(clientId)) + { + return null; + } + + var id = clientId.Trim(); + return await collection.Find(c => c.ClientId == id) + .FirstOrDefaultAsync(cancellationToken) + .ConfigureAwait(false); + } + + public async ValueTask UpsertAsync(AuthorityClientDocument document, CancellationToken cancellationToken) + { + ArgumentNullException.ThrowIfNull(document); + + document.UpdatedAt = clock.GetUtcNow(); + + var filter = Builders.Filter.Eq(c => c.ClientId, document.ClientId); + var options = new ReplaceOptions { IsUpsert = true }; + + var result = await collection.ReplaceOneAsync(filter, document, options, cancellationToken).ConfigureAwait(false); + + if (result.UpsertedId is not null) + { + logger.LogInformation("Inserted Authority client {ClientId}.", document.ClientId); + } + } + + public async ValueTask DeleteByClientIdAsync(string clientId, CancellationToken cancellationToken) + { + if (string.IsNullOrWhiteSpace(clientId)) + { + return false; + } + + var id = clientId.Trim(); + var result = await collection.DeleteOneAsync(c => c.ClientId == id, cancellationToken).ConfigureAwait(false); + return result.DeletedCount > 0; + } +} diff --git a/src/StellaOps.Authority/StellaOps.Authority.Storage.Mongo/Stores/AuthorityLoginAttemptStore.cs b/src/StellaOps.Authority/StellaOps.Authority.Storage.Mongo/Stores/AuthorityLoginAttemptStore.cs new file mode 100644 index 00000000..d9530ccd --- /dev/null +++ b/src/StellaOps.Authority/StellaOps.Authority.Storage.Mongo/Stores/AuthorityLoginAttemptStore.cs @@ -0,0 +1,51 @@ +using Microsoft.Extensions.Logging; +using MongoDB.Driver; +using StellaOps.Authority.Storage.Mongo.Documents; + +namespace StellaOps.Authority.Storage.Mongo.Stores; + +internal sealed class AuthorityLoginAttemptStore : IAuthorityLoginAttemptStore +{ + private readonly IMongoCollection collection; + private readonly ILogger logger; + + public AuthorityLoginAttemptStore( + IMongoCollection collection, + ILogger logger) + { + this.collection = collection ?? throw new ArgumentNullException(nameof(collection)); + this.logger = logger ?? throw new ArgumentNullException(nameof(logger)); + } + + public async ValueTask InsertAsync(AuthorityLoginAttemptDocument document, CancellationToken cancellationToken) + { + ArgumentNullException.ThrowIfNull(document); + + await collection.InsertOneAsync(document, cancellationToken: cancellationToken).ConfigureAwait(false); + logger.LogDebug( + "Recorded login attempt for subject '{SubjectId}' (success={Successful}).", + document.SubjectId ?? document.Username ?? "", + document.Successful); + } + + public async ValueTask> ListRecentAsync(string subjectId, int limit, CancellationToken cancellationToken) + { + if (string.IsNullOrWhiteSpace(subjectId) || limit <= 0) + { + return Array.Empty(); + } + + var normalized = subjectId.Trim(); + + var cursor = await collection.FindAsync( + Builders.Filter.Eq(a => a.SubjectId, normalized), + new FindOptions + { + Sort = Builders.Sort.Descending(a => a.OccurredAt), + Limit = limit + }, + cancellationToken).ConfigureAwait(false); + + return await cursor.ToListAsync(cancellationToken).ConfigureAwait(false); + } +} diff --git a/src/StellaOps.Authority/StellaOps.Authority.Storage.Mongo/Stores/AuthorityScopeStore.cs b/src/StellaOps.Authority/StellaOps.Authority.Storage.Mongo/Stores/AuthorityScopeStore.cs new file mode 100644 index 00000000..c8f52f38 --- /dev/null +++ b/src/StellaOps.Authority/StellaOps.Authority.Storage.Mongo/Stores/AuthorityScopeStore.cs @@ -0,0 +1,69 @@ +using Microsoft.Extensions.Logging; +using MongoDB.Driver; +using StellaOps.Authority.Storage.Mongo.Documents; + +namespace StellaOps.Authority.Storage.Mongo.Stores; + +internal sealed class AuthorityScopeStore : IAuthorityScopeStore +{ + private readonly IMongoCollection collection; + private readonly TimeProvider clock; + private readonly ILogger logger; + + public AuthorityScopeStore( + IMongoCollection collection, + TimeProvider clock, + ILogger logger) + { + this.collection = collection ?? throw new ArgumentNullException(nameof(collection)); + this.clock = clock ?? throw new ArgumentNullException(nameof(clock)); + this.logger = logger ?? throw new ArgumentNullException(nameof(logger)); + } + + public async ValueTask FindByNameAsync(string name, CancellationToken cancellationToken) + { + if (string.IsNullOrWhiteSpace(name)) + { + return null; + } + + var normalized = name.Trim(); + return await collection.Find(s => s.Name == normalized) + .FirstOrDefaultAsync(cancellationToken) + .ConfigureAwait(false); + } + + public async ValueTask> ListAsync(CancellationToken cancellationToken) + { + var cursor = await collection.FindAsync(FilterDefinition.Empty, cancellationToken: cancellationToken).ConfigureAwait(false); + return await cursor.ToListAsync(cancellationToken).ConfigureAwait(false); + } + + public async ValueTask UpsertAsync(AuthorityScopeDocument document, CancellationToken cancellationToken) + { + ArgumentNullException.ThrowIfNull(document); + + document.UpdatedAt = clock.GetUtcNow(); + + var filter = Builders.Filter.Eq(s => s.Name, document.Name); + var options = new ReplaceOptions { IsUpsert = true }; + + var result = await collection.ReplaceOneAsync(filter, document, options, cancellationToken).ConfigureAwait(false); + if (result.UpsertedId is not null) + { + logger.LogInformation("Inserted Authority scope {ScopeName}.", document.Name); + } + } + + public async ValueTask DeleteByNameAsync(string name, CancellationToken cancellationToken) + { + if (string.IsNullOrWhiteSpace(name)) + { + return false; + } + + var normalized = name.Trim(); + var result = await collection.DeleteOneAsync(s => s.Name == normalized, cancellationToken).ConfigureAwait(false); + return result.DeletedCount > 0; + } +} diff --git a/src/StellaOps.Authority/StellaOps.Authority.Storage.Mongo/Stores/AuthorityTokenStore.cs b/src/StellaOps.Authority/StellaOps.Authority.Storage.Mongo/Stores/AuthorityTokenStore.cs new file mode 100644 index 00000000..4f6659c1 --- /dev/null +++ b/src/StellaOps.Authority/StellaOps.Authority.Storage.Mongo/Stores/AuthorityTokenStore.cs @@ -0,0 +1,93 @@ +using Microsoft.Extensions.Logging; +using MongoDB.Driver; +using StellaOps.Authority.Storage.Mongo.Documents; + +namespace StellaOps.Authority.Storage.Mongo.Stores; + +internal sealed class AuthorityTokenStore : IAuthorityTokenStore +{ + private readonly IMongoCollection collection; + private readonly ILogger logger; + + public AuthorityTokenStore( + IMongoCollection collection, + ILogger logger) + { + this.collection = collection ?? throw new ArgumentNullException(nameof(collection)); + this.logger = logger ?? throw new ArgumentNullException(nameof(logger)); + } + + public async ValueTask InsertAsync(AuthorityTokenDocument document, CancellationToken cancellationToken) + { + ArgumentNullException.ThrowIfNull(document); + + await collection.InsertOneAsync(document, cancellationToken: cancellationToken).ConfigureAwait(false); + logger.LogDebug("Inserted Authority token {TokenId}.", document.TokenId); + } + + public async ValueTask FindByTokenIdAsync(string tokenId, CancellationToken cancellationToken) + { + if (string.IsNullOrWhiteSpace(tokenId)) + { + return null; + } + + var id = tokenId.Trim(); + return await collection.Find(t => t.TokenId == id) + .FirstOrDefaultAsync(cancellationToken) + .ConfigureAwait(false); + } + + public async ValueTask FindByReferenceIdAsync(string referenceId, CancellationToken cancellationToken) + { + if (string.IsNullOrWhiteSpace(referenceId)) + { + return null; + } + + var id = referenceId.Trim(); + return await collection.Find(t => t.ReferenceId == id) + .FirstOrDefaultAsync(cancellationToken) + .ConfigureAwait(false); + } + + public async ValueTask UpdateStatusAsync(string tokenId, string status, DateTimeOffset? revokedAt, CancellationToken cancellationToken) + { + if (string.IsNullOrWhiteSpace(tokenId)) + { + throw new ArgumentException("Token id cannot be empty.", nameof(tokenId)); + } + + if (string.IsNullOrWhiteSpace(status)) + { + throw new ArgumentException("Status cannot be empty.", nameof(status)); + } + + var update = Builders.Update + .Set(t => t.Status, status) + .Set(t => t.RevokedAt, revokedAt); + + var result = await collection.UpdateOneAsync( + Builders.Filter.Eq(t => t.TokenId, tokenId.Trim()), + update, + cancellationToken: cancellationToken).ConfigureAwait(false); + + logger.LogDebug("Updated token {TokenId} status to {Status} (matched {Matched}).", tokenId, status, result.MatchedCount); + } + + public async ValueTask DeleteExpiredAsync(DateTimeOffset threshold, CancellationToken cancellationToken) + { + var filter = Builders.Filter.And( + Builders.Filter.Not( + Builders.Filter.Eq(t => t.Status, "revoked")), + Builders.Filter.Lt(t => t.ExpiresAt, threshold)); + + var result = await collection.DeleteManyAsync(filter, cancellationToken).ConfigureAwait(false); + if (result.DeletedCount > 0) + { + logger.LogInformation("Deleted {Count} expired Authority tokens.", result.DeletedCount); + } + + return result.DeletedCount; + } +} diff --git a/src/StellaOps.Authority/StellaOps.Authority.Storage.Mongo/Stores/AuthorityUserStore.cs b/src/StellaOps.Authority/StellaOps.Authority.Storage.Mongo/Stores/AuthorityUserStore.cs new file mode 100644 index 00000000..03242665 --- /dev/null +++ b/src/StellaOps.Authority/StellaOps.Authority.Storage.Mongo/Stores/AuthorityUserStore.cs @@ -0,0 +1,81 @@ +using Microsoft.Extensions.Logging; +using MongoDB.Driver; +using StellaOps.Authority.Storage.Mongo.Documents; + +namespace StellaOps.Authority.Storage.Mongo.Stores; + +internal sealed class AuthorityUserStore : IAuthorityUserStore +{ + private readonly IMongoCollection collection; + private readonly TimeProvider clock; + private readonly ILogger logger; + + public AuthorityUserStore( + IMongoCollection collection, + TimeProvider clock, + ILogger logger) + { + this.collection = collection ?? throw new ArgumentNullException(nameof(collection)); + this.clock = clock ?? throw new ArgumentNullException(nameof(clock)); + this.logger = logger ?? throw new ArgumentNullException(nameof(logger)); + } + + public async ValueTask FindBySubjectIdAsync(string subjectId, CancellationToken cancellationToken) + { + if (string.IsNullOrWhiteSpace(subjectId)) + { + return null; + } + + return await collection + .Find(u => u.SubjectId == subjectId) + .FirstOrDefaultAsync(cancellationToken) + .ConfigureAwait(false); + } + + public async ValueTask FindByNormalizedUsernameAsync(string normalizedUsername, CancellationToken cancellationToken) + { + if (string.IsNullOrWhiteSpace(normalizedUsername)) + { + return null; + } + + var normalised = normalizedUsername.Trim(); + + return await collection + .Find(u => u.NormalizedUsername == normalised) + .FirstOrDefaultAsync(cancellationToken) + .ConfigureAwait(false); + } + + public async ValueTask UpsertAsync(AuthorityUserDocument document, CancellationToken cancellationToken) + { + ArgumentNullException.ThrowIfNull(document); + + document.UpdatedAt = clock.GetUtcNow(); + + var filter = Builders.Filter.Eq(u => u.SubjectId, document.SubjectId); + var options = new ReplaceOptions { IsUpsert = true }; + + var result = await collection + .ReplaceOneAsync(filter, document, options, cancellationToken) + .ConfigureAwait(false); + + if (result.UpsertedId is not null) + { + logger.LogInformation("Inserted Authority user {SubjectId}.", document.SubjectId); + } + } + + public async ValueTask DeleteBySubjectIdAsync(string subjectId, CancellationToken cancellationToken) + { + if (string.IsNullOrWhiteSpace(subjectId)) + { + return false; + } + + var normalised = subjectId.Trim(); + var result = await collection.DeleteOneAsync(u => u.SubjectId == normalised, cancellationToken).ConfigureAwait(false); + return result.DeletedCount > 0; + } +} diff --git a/src/StellaOps.Authority/StellaOps.Authority.Storage.Mongo/Stores/IAuthorityClientStore.cs b/src/StellaOps.Authority/StellaOps.Authority.Storage.Mongo/Stores/IAuthorityClientStore.cs new file mode 100644 index 00000000..67778ab8 --- /dev/null +++ b/src/StellaOps.Authority/StellaOps.Authority.Storage.Mongo/Stores/IAuthorityClientStore.cs @@ -0,0 +1,12 @@ +using StellaOps.Authority.Storage.Mongo.Documents; + +namespace StellaOps.Authority.Storage.Mongo.Stores; + +public interface IAuthorityClientStore +{ + ValueTask FindByClientIdAsync(string clientId, CancellationToken cancellationToken); + + ValueTask UpsertAsync(AuthorityClientDocument document, CancellationToken cancellationToken); + + ValueTask DeleteByClientIdAsync(string clientId, CancellationToken cancellationToken); +} diff --git a/src/StellaOps.Authority/StellaOps.Authority.Storage.Mongo/Stores/IAuthorityLoginAttemptStore.cs b/src/StellaOps.Authority/StellaOps.Authority.Storage.Mongo/Stores/IAuthorityLoginAttemptStore.cs new file mode 100644 index 00000000..f97d884b --- /dev/null +++ b/src/StellaOps.Authority/StellaOps.Authority.Storage.Mongo/Stores/IAuthorityLoginAttemptStore.cs @@ -0,0 +1,10 @@ +using StellaOps.Authority.Storage.Mongo.Documents; + +namespace StellaOps.Authority.Storage.Mongo.Stores; + +public interface IAuthorityLoginAttemptStore +{ + ValueTask InsertAsync(AuthorityLoginAttemptDocument document, CancellationToken cancellationToken); + + ValueTask> ListRecentAsync(string subjectId, int limit, CancellationToken cancellationToken); +} diff --git a/src/StellaOps.Authority/StellaOps.Authority.Storage.Mongo/Stores/IAuthorityScopeStore.cs b/src/StellaOps.Authority/StellaOps.Authority.Storage.Mongo/Stores/IAuthorityScopeStore.cs new file mode 100644 index 00000000..f51cdc87 --- /dev/null +++ b/src/StellaOps.Authority/StellaOps.Authority.Storage.Mongo/Stores/IAuthorityScopeStore.cs @@ -0,0 +1,14 @@ +using StellaOps.Authority.Storage.Mongo.Documents; + +namespace StellaOps.Authority.Storage.Mongo.Stores; + +public interface IAuthorityScopeStore +{ + ValueTask FindByNameAsync(string name, CancellationToken cancellationToken); + + ValueTask> ListAsync(CancellationToken cancellationToken); + + ValueTask UpsertAsync(AuthorityScopeDocument document, CancellationToken cancellationToken); + + ValueTask DeleteByNameAsync(string name, CancellationToken cancellationToken); +} diff --git a/src/StellaOps.Authority/StellaOps.Authority.Storage.Mongo/Stores/IAuthorityTokenStore.cs b/src/StellaOps.Authority/StellaOps.Authority.Storage.Mongo/Stores/IAuthorityTokenStore.cs new file mode 100644 index 00000000..053d97cf --- /dev/null +++ b/src/StellaOps.Authority/StellaOps.Authority.Storage.Mongo/Stores/IAuthorityTokenStore.cs @@ -0,0 +1,16 @@ +using StellaOps.Authority.Storage.Mongo.Documents; + +namespace StellaOps.Authority.Storage.Mongo.Stores; + +public interface IAuthorityTokenStore +{ + ValueTask InsertAsync(AuthorityTokenDocument document, CancellationToken cancellationToken); + + ValueTask FindByTokenIdAsync(string tokenId, CancellationToken cancellationToken); + + ValueTask FindByReferenceIdAsync(string referenceId, CancellationToken cancellationToken); + + ValueTask UpdateStatusAsync(string tokenId, string status, DateTimeOffset? revokedAt, CancellationToken cancellationToken); + + ValueTask DeleteExpiredAsync(DateTimeOffset threshold, CancellationToken cancellationToken); +} diff --git a/src/StellaOps.Authority/StellaOps.Authority.Storage.Mongo/Stores/IAuthorityUserStore.cs b/src/StellaOps.Authority/StellaOps.Authority.Storage.Mongo/Stores/IAuthorityUserStore.cs new file mode 100644 index 00000000..6f7cdf55 --- /dev/null +++ b/src/StellaOps.Authority/StellaOps.Authority.Storage.Mongo/Stores/IAuthorityUserStore.cs @@ -0,0 +1,14 @@ +using StellaOps.Authority.Storage.Mongo.Documents; + +namespace StellaOps.Authority.Storage.Mongo.Stores; + +public interface IAuthorityUserStore +{ + ValueTask FindBySubjectIdAsync(string subjectId, CancellationToken cancellationToken); + + ValueTask FindByNormalizedUsernameAsync(string normalizedUsername, CancellationToken cancellationToken); + + ValueTask UpsertAsync(AuthorityUserDocument document, CancellationToken cancellationToken); + + ValueTask DeleteBySubjectIdAsync(string subjectId, CancellationToken cancellationToken); +} diff --git a/src/StellaOps.Authority/StellaOps.Authority.Tests/Identity/AuthorityIdentityProviderRegistryTests.cs b/src/StellaOps.Authority/StellaOps.Authority.Tests/Identity/AuthorityIdentityProviderRegistryTests.cs new file mode 100644 index 00000000..a1f2a874 --- /dev/null +++ b/src/StellaOps.Authority/StellaOps.Authority.Tests/Identity/AuthorityIdentityProviderRegistryTests.cs @@ -0,0 +1,125 @@ +using Microsoft.Extensions.Configuration; +using Microsoft.Extensions.Logging.Abstractions; +using StellaOps.Authority.Plugins.Abstractions; +using Xunit; +using System.Linq; + +namespace StellaOps.Authority.Tests.Identity; + +public class AuthorityIdentityProviderRegistryTests +{ + [Fact] + public void RegistryIndexesProvidersAndAggregatesCapabilities() + { + var providers = new[] + { + CreateProvider("standard", type: "standard", supportsPassword: true, supportsMfa: false, supportsClientProvisioning: false), + CreateProvider("sso", type: "saml", supportsPassword: false, supportsMfa: true, supportsClientProvisioning: true) + }; + + var registry = new AuthorityIdentityProviderRegistry(providers, NullLogger.Instance); + + Assert.Equal(2, registry.Providers.Count); + Assert.True(registry.TryGet("standard", out var standard)); + Assert.Same(providers[0], standard); + Assert.Single(registry.PasswordProviders); + Assert.Single(registry.MfaProviders); + Assert.Single(registry.ClientProvisioningProviders); + Assert.True(registry.AggregateCapabilities.SupportsPassword); + Assert.True(registry.AggregateCapabilities.SupportsMfa); + Assert.True(registry.AggregateCapabilities.SupportsClientProvisioning); + } + + [Fact] + public void RegistryIgnoresDuplicateNames() + { + var duplicate = CreateProvider("standard", "ldap", supportsPassword: true, supportsMfa: false, supportsClientProvisioning: false); + var providers = new[] + { + CreateProvider("standard", type: "standard", supportsPassword: true, supportsMfa: false, supportsClientProvisioning: false), + duplicate + }; + + var registry = new AuthorityIdentityProviderRegistry(providers, NullLogger.Instance); + + Assert.Single(registry.Providers); + Assert.Same(providers[0], registry.Providers.First()); + Assert.True(registry.TryGet("standard", out var provider)); + Assert.Same(providers[0], provider); + } + + private static IIdentityProviderPlugin CreateProvider( + string name, + string type, + bool supportsPassword, + bool supportsMfa, + bool supportsClientProvisioning) + { + var manifest = new AuthorityPluginManifest( + name, + type, + true, + AssemblyName: null, + AssemblyPath: null, + Capabilities: BuildCapabilities(supportsPassword, supportsMfa, supportsClientProvisioning), + Metadata: new Dictionary(StringComparer.OrdinalIgnoreCase), + ConfigPath: string.Empty); + + var context = new AuthorityPluginContext(manifest, new ConfigurationBuilder().Build()); + return new TestIdentityProviderPlugin(context, supportsPassword, supportsMfa, supportsClientProvisioning); + } + + private static IReadOnlyList BuildCapabilities(bool password, bool mfa, bool clientProvisioning) + { + var capabilities = new List(); + if (password) + { + capabilities.Add(AuthorityPluginCapabilities.Password); + } + + if (mfa) + { + capabilities.Add(AuthorityPluginCapabilities.Mfa); + } + + if (clientProvisioning) + { + capabilities.Add(AuthorityPluginCapabilities.ClientProvisioning); + } + + return capabilities; + } + + private sealed class TestIdentityProviderPlugin : IIdentityProviderPlugin + { + public TestIdentityProviderPlugin( + AuthorityPluginContext context, + bool supportsPassword, + bool supportsMfa, + bool supportsClientProvisioning) + { + Context = context; + Capabilities = new AuthorityIdentityProviderCapabilities( + SupportsPassword: supportsPassword, + SupportsMfa: supportsMfa, + SupportsClientProvisioning: supportsClientProvisioning); + } + + public string Name => Context.Manifest.Name; + + public string Type => Context.Manifest.Type; + + public AuthorityPluginContext Context { get; } + + public IUserCredentialStore Credentials => throw new NotImplementedException(); + + public IClaimsEnricher ClaimsEnricher => throw new NotImplementedException(); + + public IClientProvisioningStore? ClientProvisioning => null; + + public AuthorityIdentityProviderCapabilities Capabilities { get; } + + public ValueTask CheckHealthAsync(CancellationToken cancellationToken) + => ValueTask.FromResult(AuthorityPluginHealthResult.Healthy()); + } +} diff --git a/src/StellaOps.Authority/StellaOps.Authority.Tests/Identity/AuthorityIdentityProviderSelectorTests.cs b/src/StellaOps.Authority/StellaOps.Authority.Tests/Identity/AuthorityIdentityProviderSelectorTests.cs new file mode 100644 index 00000000..83decc23 --- /dev/null +++ b/src/StellaOps.Authority/StellaOps.Authority.Tests/Identity/AuthorityIdentityProviderSelectorTests.cs @@ -0,0 +1,118 @@ +using Microsoft.Extensions.Configuration; +using OpenIddict.Abstractions; +using StellaOps.Authority.OpenIddict; +using StellaOps.Authority.Plugins.Abstractions; +using Xunit; + +namespace StellaOps.Authority.Tests.Identity; + +public class AuthorityIdentityProviderSelectorTests +{ + [Fact] + public void ResolvePasswordProvider_UsesSingleProviderWhenNoParameter() + { + var registry = CreateRegistry(passwordProviders: new[] { CreateProvider("standard", supportsPassword: true) }); + var request = new OpenIddictRequest(); + + var result = AuthorityIdentityProviderSelector.ResolvePasswordProvider(request, registry); + + Assert.True(result.Succeeded); + Assert.Equal("standard", result.Provider!.Name); + } + + [Fact] + public void ResolvePasswordProvider_FailsWhenNoProviders() + { + var registry = CreateRegistry(passwordProviders: Array.Empty()); + var request = new OpenIddictRequest(); + + var result = AuthorityIdentityProviderSelector.ResolvePasswordProvider(request, registry); + + Assert.False(result.Succeeded); + Assert.Equal(OpenIddictConstants.Errors.UnsupportedGrantType, result.Error); + } + + [Fact] + public void ResolvePasswordProvider_RequiresParameterWhenMultipleProviders() + { + var registry = CreateRegistry(passwordProviders: new[] + { + CreateProvider("standard", supportsPassword: true), + CreateProvider("ldap", supportsPassword: true) + }); + var request = new OpenIddictRequest(); + + var result = AuthorityIdentityProviderSelector.ResolvePasswordProvider(request, registry); + + Assert.False(result.Succeeded); + Assert.Equal(OpenIddictConstants.Errors.InvalidRequest, result.Error); + } + + [Fact] + public void ResolvePasswordProvider_HonoursProviderParameter() + { + var registry = CreateRegistry(passwordProviders: new[] + { + CreateProvider("standard", supportsPassword: true), + CreateProvider("ldap", supportsPassword: true) + }); + var request = new OpenIddictRequest(); + request.SetParameter(AuthorityOpenIddictConstants.ProviderParameterName, "ldap"); + + var result = AuthorityIdentityProviderSelector.ResolvePasswordProvider(request, registry); + + Assert.True(result.Succeeded); + Assert.Equal("ldap", result.Provider!.Name); + } + + private static AuthorityIdentityProviderRegistry CreateRegistry(IEnumerable passwordProviders) + { + var providers = passwordProviders.ToList(); + return new AuthorityIdentityProviderRegistry(providers, Microsoft.Extensions.Logging.Abstractions.NullLogger.Instance); + } + + private static IIdentityProviderPlugin CreateProvider(string name, bool supportsPassword) + { + var manifest = new AuthorityPluginManifest( + name, + "standard", + true, + AssemblyName: null, + AssemblyPath: null, + Capabilities: supportsPassword ? new[] { AuthorityPluginCapabilities.Password } : Array.Empty(), + Metadata: new Dictionary(StringComparer.OrdinalIgnoreCase), + ConfigPath: string.Empty); + + var context = new AuthorityPluginContext(manifest, new ConfigurationBuilder().Build()); + return new SelectorTestProvider(context, supportsPassword); + } + + private sealed class SelectorTestProvider : IIdentityProviderPlugin + { + public SelectorTestProvider(AuthorityPluginContext context, bool supportsPassword) + { + Context = context; + Capabilities = new AuthorityIdentityProviderCapabilities( + SupportsPassword: supportsPassword, + SupportsMfa: false, + SupportsClientProvisioning: false); + } + + public string Name => Context.Manifest.Name; + + public string Type => Context.Manifest.Type; + + public AuthorityPluginContext Context { get; } + + public IUserCredentialStore Credentials => throw new NotImplementedException(); + + public IClaimsEnricher ClaimsEnricher => throw new NotImplementedException(); + + public IClientProvisioningStore? ClientProvisioning => null; + + public AuthorityIdentityProviderCapabilities Capabilities { get; } + + public ValueTask CheckHealthAsync(CancellationToken cancellationToken) + => ValueTask.FromResult(AuthorityPluginHealthResult.Healthy()); + } +} diff --git a/src/StellaOps.Authority/StellaOps.Authority.Tests/OpenIddict/ClientCredentialsAndTokenHandlersTests.cs b/src/StellaOps.Authority/StellaOps.Authority.Tests/OpenIddict/ClientCredentialsAndTokenHandlersTests.cs new file mode 100644 index 00000000..46d6974c --- /dev/null +++ b/src/StellaOps.Authority/StellaOps.Authority.Tests/OpenIddict/ClientCredentialsAndTokenHandlersTests.cs @@ -0,0 +1,449 @@ +using System.Diagnostics; +using System.Security.Claims; +using Microsoft.Extensions.Configuration; +using Microsoft.Extensions.Logging.Abstractions; +using OpenIddict.Abstractions; +using OpenIddict.Extensions; +using OpenIddict.Server; +using OpenIddict.Server.AspNetCore; +using StellaOps.Auth.Abstractions; +using StellaOps.Authority.OpenIddict; +using StellaOps.Authority.OpenIddict.Handlers; +using StellaOps.Authority.Plugins.Abstractions; +using StellaOps.Authority.Storage.Mongo.Documents; +using StellaOps.Authority.Storage.Mongo.Stores; +using Xunit; +using static StellaOps.Authority.Tests.OpenIddict.TestHelpers; + +namespace StellaOps.Authority.Tests.OpenIddict; + +public class ClientCredentialsHandlersTests +{ + private static readonly ActivitySource TestActivitySource = new("StellaOps.Authority.Tests"); + + [Fact] + public async Task ValidateClientCredentials_Rejects_WhenScopeNotAllowed() + { + var clientDocument = CreateClient( + secret: "s3cr3t!", + allowedGrantTypes: "client_credentials", + allowedScopes: "jobs:read"); + + var registry = CreateRegistry(withClientProvisioning: true, clientDescriptor: CreateDescriptor(clientDocument)); + var handler = new ValidateClientCredentialsHandler(new TestClientStore(clientDocument), registry, TestActivitySource, NullLogger.Instance); + + var transaction = CreateTokenTransaction(clientDocument.ClientId, "s3cr3t!", scope: "jobs:write"); + var context = new OpenIddictServerEvents.ValidateTokenRequestContext(transaction); + + await handler.HandleAsync(context); + + Assert.True(context.IsRejected); + Assert.Equal(OpenIddictConstants.Errors.InvalidScope, context.Error); + Assert.Equal("Scope 'jobs:write' is not allowed for this client.", context.ErrorDescription); + } + + [Fact] + public async Task ValidateClientCredentials_Allows_WhenConfigurationMatches() + { + var clientDocument = CreateClient( + secret: "s3cr3t!", + allowedGrantTypes: "client_credentials", + allowedScopes: "jobs:read jobs:trigger"); + + var registry = CreateRegistry(withClientProvisioning: true, clientDescriptor: CreateDescriptor(clientDocument)); + var handler = new ValidateClientCredentialsHandler(new TestClientStore(clientDocument), registry, TestActivitySource, NullLogger.Instance); + + var transaction = CreateTokenTransaction(clientDocument.ClientId, "s3cr3t!", scope: "jobs:read"); + var context = new OpenIddictServerEvents.ValidateTokenRequestContext(transaction); + + await handler.HandleAsync(context); + + Assert.False(context.IsRejected); + Assert.Same(clientDocument, context.Transaction.Properties[AuthorityOpenIddictConstants.ClientTransactionProperty]); + + var grantedScopes = Assert.IsType(context.Transaction.Properties[AuthorityOpenIddictConstants.ClientGrantedScopesProperty]); + Assert.Equal(new[] { "jobs:read" }, grantedScopes); + Assert.Equal(clientDocument.Plugin, context.Transaction.Properties[AuthorityOpenIddictConstants.ClientProviderTransactionProperty]); + } + + [Fact] + public async Task HandleClientCredentials_PersistsTokenAndEnrichesClaims() + { + var clientDocument = CreateClient( + secret: null, + clientType: "public", + allowedGrantTypes: "client_credentials", + allowedScopes: "jobs:trigger"); + + var descriptor = CreateDescriptor(clientDocument); + var registry = CreateRegistry(withClientProvisioning: true, clientDescriptor: descriptor); + var tokenStore = new TestTokenStore(); + var handler = new HandleClientCredentialsHandler(registry, tokenStore, TimeProvider.System, TestActivitySource, NullLogger.Instance); + + var transaction = CreateTokenTransaction(clientDocument.ClientId, secret: null, scope: "jobs:trigger"); + transaction.Options.AccessTokenLifetime = TimeSpan.FromMinutes(30); + transaction.Properties[AuthorityOpenIddictConstants.ClientTransactionProperty] = clientDocument; + transaction.Properties[AuthorityOpenIddictConstants.ClientProviderTransactionProperty] = clientDocument.Plugin!; + transaction.Properties[AuthorityOpenIddictConstants.ClientGrantedScopesProperty] = new[] { "jobs:trigger" }; + + var context = new OpenIddictServerEvents.HandleTokenRequestContext(transaction); + + await handler.HandleAsync(context); + + Assert.True(context.IsRequestHandled); + Assert.NotNull(context.Principal); + + var identityProviderClaim = context.Principal?.GetClaim(StellaOpsClaimTypes.IdentityProvider); + Assert.Equal(clientDocument.Plugin, identityProviderClaim); + + var tokenId = context.Principal?.GetClaim(OpenIddictConstants.Claims.JwtId); + Assert.False(string.IsNullOrWhiteSpace(tokenId)); + + var persisted = Assert.IsType(tokenStore.Inserted); + Assert.Equal(tokenId, persisted.TokenId); + Assert.Equal(clientDocument.ClientId, persisted.ClientId); + Assert.Equal("valid", persisted.Status); + Assert.Equal(new[] { "jobs:trigger" }, persisted.Scope); + } +} + +public class TokenValidationHandlersTests +{ + private static readonly ActivitySource TestActivitySource = new("StellaOps.Authority.Tests.TokenValidation"); + + [Fact] + public async Task ValidateAccessTokenHandler_Rejects_WhenTokenRevoked() + { + var tokenStore = new TestTokenStore(); + tokenStore.Inserted = new AuthorityTokenDocument + { + TokenId = "token-1", + Status = "revoked", + ClientId = "feedser" + }; + + var handler = new ValidateAccessTokenHandler( + tokenStore, + new TestClientStore(CreateClient()), + CreateRegistry(withClientProvisioning: true, clientDescriptor: CreateDescriptor(CreateClient())), + TimeProvider.System, + TestActivitySource, + NullLogger.Instance); + + var transaction = new OpenIddictServerTransaction + { + Options = new OpenIddictServerOptions(), + EndpointType = OpenIddictServerEndpointType.Token, + Request = new OpenIddictRequest() + }; + + var principal = CreatePrincipal("feedser", "token-1", "standard"); + var context = new OpenIddictServerEvents.ValidateTokenContext(transaction) + { + Principal = principal, + TokenId = "token-1" + }; + + await handler.HandleAsync(context); + + Assert.True(context.IsRejected); + Assert.Equal(OpenIddictConstants.Errors.InvalidToken, context.Error); + } + + [Fact] + public async Task ValidateAccessTokenHandler_EnrichesClaims_WhenProviderAvailable() + { + var clientDocument = CreateClient(); + var userDescriptor = new AuthorityUserDescriptor("user-1", "alice", displayName: "Alice", requiresPasswordReset: false); + + var plugin = CreatePlugin( + name: "standard", + supportsClientProvisioning: true, + descriptor: CreateDescriptor(clientDocument), + user: userDescriptor); + + var registry = new AuthorityIdentityProviderRegistry(new[] { plugin }, NullLogger.Instance); + + var handler = new ValidateAccessTokenHandler( + new TestTokenStore(), + new TestClientStore(clientDocument), + registry, + TimeProvider.System, + TestActivitySource, + NullLogger.Instance); + + var transaction = new OpenIddictServerTransaction + { + Options = new OpenIddictServerOptions(), + EndpointType = OpenIddictServerEndpointType.Token, + Request = new OpenIddictRequest() + }; + + var principal = CreatePrincipal(clientDocument.ClientId, "token-123", plugin.Name, subject: userDescriptor.SubjectId); + var context = new OpenIddictServerEvents.ValidateTokenContext(transaction) + { + Principal = principal + }; + + await handler.HandleAsync(context); + + Assert.False(context.IsRejected); + Assert.Contains(principal.Claims, claim => claim.Type == "enriched" && claim.Value == "true"); + } +} + +internal sealed class TestClientStore : IAuthorityClientStore +{ + private readonly Dictionary clients = new(StringComparer.OrdinalIgnoreCase); + + public TestClientStore(params AuthorityClientDocument[] documents) + { + foreach (var document in documents) + { + clients[document.ClientId] = document; + } + } + + public ValueTask FindByClientIdAsync(string clientId, CancellationToken cancellationToken) + { + clients.TryGetValue(clientId, out var document); + return ValueTask.FromResult(document); + } + + public ValueTask UpsertAsync(AuthorityClientDocument document, CancellationToken cancellationToken) + { + clients[document.ClientId] = document; + return ValueTask.CompletedTask; + } + + public ValueTask DeleteByClientIdAsync(string clientId, CancellationToken cancellationToken) + => ValueTask.FromResult(clients.Remove(clientId)); +} + +internal sealed class TestTokenStore : IAuthorityTokenStore +{ + public AuthorityTokenDocument? Inserted { get; set; } + + public ValueTask InsertAsync(AuthorityTokenDocument document, CancellationToken cancellationToken) + { + Inserted = document; + return ValueTask.CompletedTask; + } + + public ValueTask FindByTokenIdAsync(string tokenId, CancellationToken cancellationToken) + => ValueTask.FromResult(Inserted is not null && string.Equals(Inserted.TokenId, tokenId, StringComparison.OrdinalIgnoreCase) ? Inserted : null); + + public ValueTask FindByReferenceIdAsync(string referenceId, CancellationToken cancellationToken) + => ValueTask.FromResult(null); + + public ValueTask UpdateStatusAsync(string tokenId, string status, DateTimeOffset? revokedAt, CancellationToken cancellationToken) + => ValueTask.CompletedTask; + + public ValueTask DeleteExpiredAsync(DateTimeOffset threshold, CancellationToken cancellationToken) + => ValueTask.FromResult(0L); +} + +internal sealed class TestClaimsEnricher : IClaimsEnricher +{ + public ValueTask EnrichAsync(ClaimsIdentity identity, AuthorityClaimsEnrichmentContext context, CancellationToken cancellationToken) + { + if (!identity.HasClaim(c => c.Type == "enriched")) + { + identity.AddClaim(new Claim("enriched", "true")); + } + + return ValueTask.CompletedTask; + } +} + +internal sealed class TestUserCredentialStore : IUserCredentialStore +{ + private readonly AuthorityUserDescriptor? user; + + public TestUserCredentialStore(AuthorityUserDescriptor? user) + { + this.user = user; + } + + public ValueTask VerifyPasswordAsync(string username, string password, CancellationToken cancellationToken) + => ValueTask.FromResult(AuthorityCredentialVerificationResult.Failure(AuthorityCredentialFailureCode.InvalidCredentials)); + + public ValueTask> UpsertUserAsync(AuthorityUserRegistration registration, CancellationToken cancellationToken) + => ValueTask.FromResult(AuthorityPluginOperationResult.Failure("unsupported", "not implemented")); + + public ValueTask FindBySubjectAsync(string subjectId, CancellationToken cancellationToken) + => ValueTask.FromResult(user); +} + +internal sealed class TestClientProvisioningStore : IClientProvisioningStore +{ + private readonly AuthorityClientDescriptor? descriptor; + + public TestClientProvisioningStore(AuthorityClientDescriptor? descriptor) + { + this.descriptor = descriptor; + } + + public ValueTask> CreateOrUpdateAsync(AuthorityClientRegistration registration, CancellationToken cancellationToken) + => ValueTask.FromResult(AuthorityPluginOperationResult.Failure("unsupported", "not implemented")); + + public ValueTask FindByClientIdAsync(string clientId, CancellationToken cancellationToken) + => ValueTask.FromResult(descriptor); + + public ValueTask DeleteAsync(string clientId, CancellationToken cancellationToken) + => ValueTask.FromResult(AuthorityPluginOperationResult.Success()); +} + +internal sealed class TestIdentityProviderPlugin : IIdentityProviderPlugin +{ + public TestIdentityProviderPlugin( + AuthorityPluginContext context, + IUserCredentialStore credentialStore, + IClaimsEnricher claimsEnricher, + IClientProvisioningStore? clientProvisioning, + AuthorityIdentityProviderCapabilities capabilities) + { + Context = context; + Credentials = credentialStore; + ClaimsEnricher = claimsEnricher; + ClientProvisioning = clientProvisioning; + Capabilities = capabilities; + } + + public string Name => Context.Manifest.Name; + + public string Type => Context.Manifest.Type; + + public AuthorityPluginContext Context { get; } + + public IUserCredentialStore Credentials { get; } + + public IClaimsEnricher ClaimsEnricher { get; } + + public IClientProvisioningStore? ClientProvisioning { get; } + + public AuthorityIdentityProviderCapabilities Capabilities { get; } + + public ValueTask CheckHealthAsync(CancellationToken cancellationToken) + => ValueTask.FromResult(AuthorityPluginHealthResult.Healthy()); +} + +internal static class TestHelpers +{ + public static AuthorityClientDocument CreateClient( + string? secret = "s3cr3t!", + string clientType = "confidential", + string allowedGrantTypes = "client_credentials", + string allowedScopes = "jobs:read") + { + return new AuthorityClientDocument + { + ClientId = "feedser", + ClientType = clientType, + SecretHash = secret is null ? null : AuthoritySecretHasher.ComputeHash(secret), + Plugin = "standard", + Properties = new Dictionary(StringComparer.OrdinalIgnoreCase) + { + [AuthorityClientMetadataKeys.AllowedGrantTypes] = allowedGrantTypes, + [AuthorityClientMetadataKeys.AllowedScopes] = allowedScopes + } + }; + } + + public static AuthorityClientDescriptor CreateDescriptor(AuthorityClientDocument document) + { + var allowedGrantTypes = document.Properties.TryGetValue(AuthorityClientMetadataKeys.AllowedGrantTypes, out var grants) ? grants?.Split(' ', StringSplitOptions.RemoveEmptyEntries) : Array.Empty(); + var allowedScopes = document.Properties.TryGetValue(AuthorityClientMetadataKeys.AllowedScopes, out var scopes) ? scopes?.Split(' ', StringSplitOptions.RemoveEmptyEntries) : Array.Empty(); + + return new AuthorityClientDescriptor( + document.ClientId, + document.DisplayName, + confidential: string.Equals(document.ClientType, "confidential", StringComparison.OrdinalIgnoreCase), + allowedGrantTypes, + allowedScopes, + redirectUris: Array.Empty(), + postLogoutRedirectUris: Array.Empty(), + properties: document.Properties); + } + + public static AuthorityIdentityProviderRegistry CreateRegistry(bool withClientProvisioning, AuthorityClientDescriptor? clientDescriptor) + { + var plugin = CreatePlugin( + name: "standard", + supportsClientProvisioning: withClientProvisioning, + descriptor: clientDescriptor, + user: null); + + return new AuthorityIdentityProviderRegistry(new[] { plugin }, NullLogger.Instance); + } + + public static TestIdentityProviderPlugin CreatePlugin( + string name, + bool supportsClientProvisioning, + AuthorityClientDescriptor? descriptor, + AuthorityUserDescriptor? user) + { + var capabilities = supportsClientProvisioning + ? new[] { AuthorityPluginCapabilities.ClientProvisioning } + : Array.Empty(); + + var manifest = new AuthorityPluginManifest( + name, + "standard", + true, + null, + null, + capabilities, + new Dictionary(StringComparer.OrdinalIgnoreCase), + $"{name}.yaml"); + + var context = new AuthorityPluginContext(manifest, new ConfigurationBuilder().Build()); + + return new TestIdentityProviderPlugin( + context, + new TestUserCredentialStore(user), + new TestClaimsEnricher(), + supportsClientProvisioning ? new TestClientProvisioningStore(descriptor) : null, + new AuthorityIdentityProviderCapabilities( + SupportsPassword: true, + SupportsMfa: false, + SupportsClientProvisioning: supportsClientProvisioning)); + } + + public static OpenIddictServerTransaction CreateTokenTransaction(string clientId, string? secret, string? scope) + { + var request = new OpenIddictRequest + { + GrantType = OpenIddictConstants.GrantTypes.ClientCredentials, + ClientId = clientId, + ClientSecret = secret + }; + + if (!string.IsNullOrWhiteSpace(scope)) + { + request.Scope = scope; + } + + return new OpenIddictServerTransaction + { + EndpointType = OpenIddictServerEndpointType.Token, + Options = new OpenIddictServerOptions(), + Request = request + }; + } + + public static ClaimsPrincipal CreatePrincipal(string clientId, string tokenId, string provider, string? subject = null) + { + var identity = new ClaimsIdentity(OpenIddictServerAspNetCoreDefaults.AuthenticationScheme); + identity.AddClaim(new Claim(OpenIddictConstants.Claims.ClientId, clientId)); + identity.AddClaim(new Claim(OpenIddictConstants.Claims.JwtId, tokenId)); + identity.AddClaim(new Claim(StellaOpsClaimTypes.IdentityProvider, provider)); + + if (!string.IsNullOrWhiteSpace(subject)) + { + identity.AddClaim(new Claim(OpenIddictConstants.Claims.Subject, subject)); + } + + return new ClaimsPrincipal(identity); + } +} diff --git a/src/StellaOps.Authority/StellaOps.Authority.Tests/OpenIddict/TokenPersistenceIntegrationTests.cs b/src/StellaOps.Authority/StellaOps.Authority.Tests/OpenIddict/TokenPersistenceIntegrationTests.cs new file mode 100644 index 00000000..ee8e3885 --- /dev/null +++ b/src/StellaOps.Authority/StellaOps.Authority.Tests/OpenIddict/TokenPersistenceIntegrationTests.cs @@ -0,0 +1,208 @@ +using System.Collections.Generic; +using System.Diagnostics; +using System.Security.Claims; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Logging.Abstractions; +using Microsoft.Extensions.Time.Testing; +using MongoDB.Driver; +using OpenIddict.Abstractions; +using OpenIddict.Server; +using StellaOps.Authority; +using StellaOps.Authority.OpenIddict; +using StellaOps.Authority.OpenIddict.Handlers; +using StellaOps.Authority.Plugins.Abstractions; +using StellaOps.Authority.Storage.Mongo; +using StellaOps.Authority.Storage.Mongo.Documents; +using StellaOps.Authority.Storage.Mongo.Extensions; +using StellaOps.Authority.Storage.Mongo.Initialization; +using StellaOps.Authority.Storage.Mongo.Stores; +using StellaOps.Feedser.Testing; +using Xunit; + +namespace StellaOps.Authority.Tests.OpenIddict; + +[Collection("mongo-fixture")] +public sealed class TokenPersistenceIntegrationTests +{ + private static readonly ActivitySource TestActivitySource = new("StellaOps.Authority.Tests.Persistence"); + private readonly MongoIntegrationFixture fixture; + + public TokenPersistenceIntegrationTests(MongoIntegrationFixture fixture) + => this.fixture = fixture ?? throw new ArgumentNullException(nameof(fixture)); + + [Fact] + public async Task HandleClientCredentials_PersistsTokenInMongo() + { + await ResetCollectionsAsync(); + + var issuedAt = new DateTimeOffset(2025, 10, 10, 12, 0, 0, TimeSpan.Zero); + var clock = new FakeTimeProvider(issuedAt); + + await using var provider = await BuildMongoProviderAsync(clock); + + var clientStore = provider.GetRequiredService(); + var tokenStore = provider.GetRequiredService(); + + var clientDocument = TestHelpers.CreateClient( + secret: "s3cr3t!", + allowedGrantTypes: "client_credentials", + allowedScopes: "jobs:trigger jobs:read"); + + await clientStore.UpsertAsync(clientDocument, CancellationToken.None); + + var registry = TestHelpers.CreateRegistry( + withClientProvisioning: true, + clientDescriptor: TestHelpers.CreateDescriptor(clientDocument)); + + var validateHandler = new ValidateClientCredentialsHandler(clientStore, registry, TestActivitySource, NullLogger.Instance); + var handleHandler = new HandleClientCredentialsHandler(registry, tokenStore, clock, TestActivitySource, NullLogger.Instance); + + var transaction = TestHelpers.CreateTokenTransaction(clientDocument.ClientId, "s3cr3t!", scope: "jobs:trigger"); + transaction.Options.AccessTokenLifetime = TimeSpan.FromMinutes(15); + + var validateContext = new OpenIddictServerEvents.ValidateTokenRequestContext(transaction); + await validateHandler.HandleAsync(validateContext); + Assert.False(validateContext.IsRejected); + + var handleContext = new OpenIddictServerEvents.HandleTokenRequestContext(transaction); + await handleHandler.HandleAsync(handleContext); + + Assert.True(handleContext.IsRequestHandled); + var principal = Assert.IsType(handleContext.Principal); + var tokenId = principal.GetClaim(OpenIddictConstants.Claims.JwtId); + Assert.False(string.IsNullOrWhiteSpace(tokenId)); + + var stored = await tokenStore.FindByTokenIdAsync(tokenId!, CancellationToken.None); + Assert.NotNull(stored); + Assert.Equal(clientDocument.ClientId, stored!.ClientId); + Assert.Equal(OpenIddictConstants.TokenTypeHints.AccessToken, stored.Type); + Assert.Equal("valid", stored.Status); + Assert.Equal(issuedAt, stored.CreatedAt); + Assert.Equal(issuedAt.AddMinutes(15), stored.ExpiresAt); + Assert.Equal(new[] { "jobs:trigger" }, stored.Scope); + } + + [Fact] + public async Task ValidateAccessTokenHandler_RejectsRevokedRefreshTokenPersistedInMongo() + { + await ResetCollectionsAsync(); + + var now = new DateTimeOffset(2025, 10, 10, 14, 0, 0, TimeSpan.Zero); + var clock = new FakeTimeProvider(now); + + await using var provider = await BuildMongoProviderAsync(clock); + + var clientStore = provider.GetRequiredService(); + var tokenStore = provider.GetRequiredService(); + + var clientDocument = TestHelpers.CreateClient( + secret: null, + clientType: "public", + allowedGrantTypes: "password refresh_token", + allowedScopes: "openid profile jobs:read"); + + await clientStore.UpsertAsync(clientDocument, CancellationToken.None); + + var descriptor = TestHelpers.CreateDescriptor(clientDocument); + var userDescriptor = new AuthorityUserDescriptor("subject-1", "alice", displayName: "Alice", requiresPasswordReset: false); + + var plugin = TestHelpers.CreatePlugin( + name: clientDocument.Plugin ?? "standard", + supportsClientProvisioning: true, + descriptor, + userDescriptor); + + var registry = new AuthorityIdentityProviderRegistry( + new[] { plugin }, + NullLogger.Instance); + + const string revokedTokenId = "refresh-token-1"; + var refreshToken = new AuthorityTokenDocument + { + TokenId = revokedTokenId, + Type = OpenIddictConstants.TokenTypeHints.RefreshToken, + SubjectId = userDescriptor.SubjectId, + ClientId = clientDocument.ClientId, + Scope = new List { "openid", "profile" }, + Status = "valid", + CreatedAt = now.AddMinutes(-5), + ExpiresAt = now.AddHours(4), + ReferenceId = "refresh-reference-1" + }; + + await tokenStore.InsertAsync(refreshToken, CancellationToken.None); + + var revokedAt = now.AddMinutes(1); + await tokenStore.UpdateStatusAsync(revokedTokenId, "revoked", revokedAt, CancellationToken.None); + + var handler = new ValidateAccessTokenHandler( + tokenStore, + clientStore, + registry, + clock, + TestActivitySource, + NullLogger.Instance); + + var transaction = new OpenIddictServerTransaction + { + EndpointType = OpenIddictServerEndpointType.Token, + Options = new OpenIddictServerOptions(), + Request = new OpenIddictRequest + { + GrantType = OpenIddictConstants.GrantTypes.RefreshToken + } + }; + + var principal = TestHelpers.CreatePrincipal( + clientDocument.ClientId, + revokedTokenId, + plugin.Name, + userDescriptor.SubjectId); + + var context = new OpenIddictServerEvents.ValidateTokenContext(transaction) + { + Principal = principal, + TokenId = revokedTokenId + }; + + await handler.HandleAsync(context); + + Assert.True(context.IsRejected); + Assert.Equal(OpenIddictConstants.Errors.InvalidToken, context.Error); + + var stored = await tokenStore.FindByTokenIdAsync(revokedTokenId, CancellationToken.None); + Assert.NotNull(stored); + Assert.Equal("revoked", stored!.Status); + Assert.Equal(revokedAt, stored.RevokedAt); + } + + private async Task ResetCollectionsAsync() + { + var tokens = fixture.Database.GetCollection(AuthorityMongoDefaults.Collections.Tokens); + await tokens.DeleteManyAsync(Builders.Filter.Empty); + + var clients = fixture.Database.GetCollection(AuthorityMongoDefaults.Collections.Clients); + await clients.DeleteManyAsync(Builders.Filter.Empty); + } + + private async Task BuildMongoProviderAsync(FakeTimeProvider clock) + { + var services = new ServiceCollection(); + services.AddSingleton(clock); + services.AddLogging(); + services.AddAuthorityMongoStorage(options => + { + options.ConnectionString = fixture.Runner.ConnectionString; + options.DatabaseName = fixture.Database.DatabaseNamespace.DatabaseName; + options.CommandTimeout = TimeSpan.FromSeconds(5); + }); + + var provider = services.BuildServiceProvider(); + + var initializer = provider.GetRequiredService(); + var database = provider.GetRequiredService(); + await initializer.InitialiseAsync(database, CancellationToken.None); + + return provider; + } +} diff --git a/src/StellaOps.Authority/StellaOps.Authority.Tests/Plugins/AuthorityPluginLoaderTests.cs b/src/StellaOps.Authority/StellaOps.Authority.Tests/Plugins/AuthorityPluginLoaderTests.cs new file mode 100644 index 00000000..ad9c22bc --- /dev/null +++ b/src/StellaOps.Authority/StellaOps.Authority.Tests/Plugins/AuthorityPluginLoaderTests.cs @@ -0,0 +1,117 @@ +using System; +using System.Collections.Generic; +using Microsoft.Extensions.Configuration; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Logging.Abstractions; +using StellaOps.Authority.Plugins; +using StellaOps.Authority.Plugins.Abstractions; + +namespace StellaOps.Authority.Tests.Plugins; + +public class AuthorityPluginLoaderTests +{ + [Fact] + public void RegisterPlugins_ReturnsEmptySummary_WhenNoPluginsConfigured() + { + var services = new ServiceCollection(); + var configuration = new ConfigurationBuilder().Build(); + + var summary = AuthorityPluginLoader.RegisterPluginsCore( + services, + configuration, + Array.Empty(), + Array.Empty(), + Array.Empty(), + NullLogger.Instance); + + Assert.Empty(summary.RegisteredPlugins); + Assert.Empty(summary.Failures); + Assert.Empty(summary.MissingOrderedPlugins); + } + + [Fact] + public void RegisterPlugins_RecordsFailure_WhenAssemblyMissing() + { + var services = new ServiceCollection(); + var hostConfiguration = new ConfigurationBuilder().Build(); + + var manifest = new AuthorityPluginManifest( + "standard", + "standard", + true, + "StellaOps.Authority.Plugin.Standard", + null, + Array.Empty(), + new Dictionary(), + "standard.yaml"); + + var contexts = new[] + { + new AuthorityPluginContext(manifest, hostConfiguration) + }; + + var summary = AuthorityPluginLoader.RegisterPluginsCore( + services, + hostConfiguration, + contexts, + Array.Empty(), + Array.Empty(), + NullLogger.Instance); + + var failure = Assert.Single(summary.Failures); + Assert.Equal("standard", failure.PluginName); + Assert.Contains("Assembly", failure.Reason, StringComparison.OrdinalIgnoreCase); + } + + [Fact] + public void RegisterPlugins_RegistersEnabledPlugin_WhenRegistrarAvailable() + { + var services = new ServiceCollection(); + var hostConfiguration = new ConfigurationBuilder().Build(); + + var manifest = new AuthorityPluginManifest( + "test", + TestAuthorityPluginRegistrar.PluginTypeIdentifier, + true, + typeof(TestAuthorityPluginRegistrar).Assembly.GetName().Name, + typeof(TestAuthorityPluginRegistrar).Assembly.Location, + Array.Empty(), + new Dictionary(), + "test.yaml"); + + var pluginContext = new AuthorityPluginContext(manifest, hostConfiguration); + var descriptor = new AuthorityPluginLoader.LoadedPluginDescriptor( + typeof(TestAuthorityPluginRegistrar).Assembly, + typeof(TestAuthorityPluginRegistrar).Assembly.Location); + + var summary = AuthorityPluginLoader.RegisterPluginsCore( + services, + hostConfiguration, + new[] { pluginContext }, + new[] { descriptor }, + Array.Empty(), + NullLogger.Instance); + + Assert.Contains("test", summary.RegisteredPlugins); + Assert.Empty(summary.Failures); + + var provider = services.BuildServiceProvider(); + Assert.NotNull(provider.GetRequiredService()); + } + + private sealed class TestAuthorityPluginRegistrar : IAuthorityPluginRegistrar + { + public const string PluginTypeIdentifier = "test-plugin"; + + public string PluginType => PluginTypeIdentifier; + + public void Register(AuthorityPluginRegistrationContext context) + { + context.Services.AddSingleton(); + } + } + + private sealed class TestMarkerService + { + } +} diff --git a/src/StellaOps.Authority/StellaOps.Authority.Tests/RateLimiting/AuthorityRateLimiterIntegrationTests.cs b/src/StellaOps.Authority/StellaOps.Authority.Tests/RateLimiting/AuthorityRateLimiterIntegrationTests.cs new file mode 100644 index 00000000..d7dc1f8c --- /dev/null +++ b/src/StellaOps.Authority/StellaOps.Authority.Tests/RateLimiting/AuthorityRateLimiterIntegrationTests.cs @@ -0,0 +1,141 @@ +using System; +using System.Collections.Generic; +using System.Net; +using System.Net.Http; +using Microsoft.AspNetCore.Builder; +using Microsoft.AspNetCore.Hosting; +using Microsoft.AspNetCore.Http; +using Microsoft.AspNetCore.TestHost; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.DependencyInjection.Extensions; +using Microsoft.Extensions.Options; +using StellaOps.Authority.RateLimiting; +using StellaOps.Configuration; +using Xunit; + +namespace StellaOps.Authority.Tests.RateLimiting; + +public class AuthorityRateLimiterIntegrationTests +{ + [Fact] + public async Task TokenEndpoint_Returns429_WhenLimitExceeded() + { + using var server = CreateServer(options => + { + options.Security.RateLimiting.Token.PermitLimit = 1; + options.Security.RateLimiting.Token.QueueLimit = 0; + options.Security.RateLimiting.Token.Window = TimeSpan.FromSeconds(30); + }); + + using var client = server.CreateClient(); + client.DefaultRequestHeaders.Add("X-Forwarded-For", "198.51.100.50"); + + var firstResponse = await client.PostAsync("/token", CreateTokenForm("feedser")); + Assert.Equal(HttpStatusCode.OK, firstResponse.StatusCode); + + var secondResponse = await client.PostAsync("/token", CreateTokenForm("feedser")); + Assert.Equal(HttpStatusCode.TooManyRequests, secondResponse.StatusCode); + Assert.NotNull(secondResponse.Headers.RetryAfter); + } + + + [Fact] + public async Task TokenEndpoint_AllowsDifferentClientIdsWithinWindow() + { + using var server = CreateServer(options => + { + options.Security.RateLimiting.Token.PermitLimit = 1; + options.Security.RateLimiting.Token.QueueLimit = 0; + options.Security.RateLimiting.Token.Window = TimeSpan.FromSeconds(30); + }); + + using var client = server.CreateClient(); + client.DefaultRequestHeaders.Add("X-Forwarded-For", "198.51.100.70"); + + var firstResponse = await client.PostAsync("/token", CreateTokenForm("alpha-client")); + Assert.Equal(HttpStatusCode.OK, firstResponse.StatusCode); + + var secondResponse = await client.PostAsync("/token", CreateTokenForm("beta-client")); + Assert.Equal(HttpStatusCode.OK, secondResponse.StatusCode); + } + + [Fact] + public async Task InternalEndpoint_Returns429_WhenLimitExceeded() + { + using var server = CreateServer(options => + { + options.Security.RateLimiting.Internal.Enabled = true; + options.Security.RateLimiting.Internal.PermitLimit = 1; + options.Security.RateLimiting.Internal.QueueLimit = 0; + options.Security.RateLimiting.Internal.Window = TimeSpan.FromSeconds(15); + }); + + using var client = server.CreateClient(); + client.DefaultRequestHeaders.Add("X-Forwarded-For", "198.51.100.60"); + + var firstResponse = await client.GetAsync("/internal/ping"); + Assert.Equal(HttpStatusCode.OK, firstResponse.StatusCode); + + var secondResponse = await client.GetAsync("/internal/ping"); + Assert.Equal(HttpStatusCode.TooManyRequests, secondResponse.StatusCode); + } + + private static TestServer CreateServer(Action? configure) + { + var options = new StellaOpsAuthorityOptions + { + Issuer = new Uri("https://authority.integration.test"), + SchemaVersion = 1 + }; + options.Storage.ConnectionString = "mongodb://localhost/authority"; + + configure?.Invoke(options); + + var builder = new WebHostBuilder() + .ConfigureServices(services => + { + services.AddSingleton(options); + services.AddSingleton>(Options.Create(options)); + services.TryAddSingleton(_ => TimeProvider.System); + services.AddHttpContextAccessor(); + services.TryAddSingleton(); + services.TryAddSingleton(); + services.AddRateLimiter(rateLimiterOptions => + { + AuthorityRateLimiter.Configure(rateLimiterOptions, options); + }); + }) + .Configure(app => + { + app.UseAuthorityRateLimiterContext(); + app.UseRateLimiter(); + + app.Map("/token", builder => + { + builder.Run(async context => + { + context.Response.StatusCode = StatusCodes.Status200OK; + await context.Response.WriteAsync("token-ok"); + }); + }); + + app.Map("/internal/ping", builder => + { + builder.Run(async context => + { + context.Response.StatusCode = StatusCodes.Status200OK; + await context.Response.WriteAsync("internal-ok"); + }); + }); + }); + + return new TestServer(builder); + } + + private static FormUrlEncodedContent CreateTokenForm(string clientId) + => new(new Dictionary + { + ["grant_type"] = "client_credentials", + ["client_id"] = clientId + }); +} diff --git a/src/StellaOps.Authority/StellaOps.Authority.Tests/RateLimiting/AuthorityRateLimiterMetadataAccessorTests.cs b/src/StellaOps.Authority/StellaOps.Authority.Tests/RateLimiting/AuthorityRateLimiterMetadataAccessorTests.cs new file mode 100644 index 00000000..be40594c --- /dev/null +++ b/src/StellaOps.Authority/StellaOps.Authority.Tests/RateLimiting/AuthorityRateLimiterMetadataAccessorTests.cs @@ -0,0 +1,30 @@ +using Microsoft.AspNetCore.Http; +using StellaOps.Authority.RateLimiting; +using Xunit; + +namespace StellaOps.Authority.Tests.RateLimiting; + +public class AuthorityRateLimiterMetadataAccessorTests +{ + [Fact] + public void SetClientId_UpdatesFeatureMetadata() + { + var context = new DefaultHttpContext(); + var feature = new AuthorityRateLimiterFeature(new AuthorityRateLimiterMetadata()); + context.Features.Set(feature); + + var accessor = new AuthorityRateLimiterMetadataAccessor(new HttpContextAccessor { HttpContext = context }); + + accessor.SetClientId("client-123"); + accessor.SetTag("custom", "tag"); + accessor.SetSubjectId("subject-1"); + + var metadata = accessor.GetMetadata(); + Assert.NotNull(metadata); + Assert.Equal("client-123", metadata!.ClientId); + Assert.Equal("subject-1", metadata.SubjectId); + Assert.Equal("client-123", metadata.Tags["authority.client_id"]); + Assert.Equal("subject-1", metadata.Tags["authority.subject_id"]); + Assert.Equal("tag", metadata.Tags["custom"]); + } +} diff --git a/src/StellaOps.Authority/StellaOps.Authority.Tests/RateLimiting/AuthorityRateLimiterMetadataMiddlewareTests.cs b/src/StellaOps.Authority/StellaOps.Authority.Tests/RateLimiting/AuthorityRateLimiterMetadataMiddlewareTests.cs new file mode 100644 index 00000000..0a072a06 --- /dev/null +++ b/src/StellaOps.Authority/StellaOps.Authority.Tests/RateLimiting/AuthorityRateLimiterMetadataMiddlewareTests.cs @@ -0,0 +1,96 @@ +using System; +using System.IO; +using System.Net; +using System.Text; +using System.Threading.Tasks; +using Microsoft.AspNetCore.Http; +using Microsoft.Extensions.Logging.Abstractions; +using StellaOps.Authority.RateLimiting; +using Xunit; + +namespace StellaOps.Authority.Tests.RateLimiting; + +public class AuthorityRateLimiterMetadataMiddlewareTests +{ + [Fact] + public async Task InvokeAsync_ExtractsClientIdFromBasicAuthorization() + { + var context = new DefaultHttpContext(); + context.Request.Path = "/token"; + context.Request.Method = HttpMethods.Post; + context.Connection.RemoteIpAddress = IPAddress.Parse("198.51.100.10"); + context.Request.Headers.Authorization = "Basic " + Convert.ToBase64String(Encoding.UTF8.GetBytes("clientA:secret")); + + var middleware = CreateMiddleware(); + await middleware.InvokeAsync(context); + + var metadata = context.Features.Get()?.Metadata; + Assert.NotNull(metadata); + Assert.Equal("/token", metadata!.Endpoint); + Assert.Equal("clientA", metadata.ClientId); + Assert.Equal("198.51.100.10", metadata.RemoteIp); + } + + [Fact] + public async Task InvokeAsync_ExtractsClientIdFromFormBody() + { + var body = "grant_type=password&client_id=form-client"; + var bytes = Encoding.UTF8.GetBytes(body); + + var context = new DefaultHttpContext(); + context.Request.Path = "/token"; + context.Request.Method = HttpMethods.Post; + context.Connection.RemoteIpAddress = IPAddress.Parse("198.51.100.20"); + context.Request.ContentType = "application/x-www-form-urlencoded"; + context.Request.Body = new MemoryStream(bytes); + context.Request.ContentLength = bytes.Length; + + var middleware = CreateMiddleware(); + await middleware.InvokeAsync(context); + + var metadata = context.Features.Get()?.Metadata; + Assert.NotNull(metadata); + Assert.Equal("form-client", metadata!.ClientId); + Assert.Equal(0, context.Request.Body.Position); + } + + [Fact] + public async Task InvokeAsync_ExtractsClientIdFromAuthorizeQuery() + { + var context = new DefaultHttpContext(); + context.Request.Path = "/authorize"; + context.Request.QueryString = new QueryString("?client_id=browser-app"); + + var middleware = CreateMiddleware(); + await middleware.InvokeAsync(context); + + var metadata = context.Features.Get()?.Metadata; + Assert.NotNull(metadata); + Assert.Equal("browser-app", metadata!.ClientId); + } + + [Fact] + public async Task InvokeAsync_PrefersForwardedFor_WhenRemoteIpMissing() + { + var context = new DefaultHttpContext(); + context.Request.Path = "/token"; + context.Request.Method = HttpMethods.Post; + context.Request.Headers["X-Forwarded-For"] = "203.0.113.99"; + + var middleware = CreateMiddleware(); + await middleware.InvokeAsync(context); + + var metadata = context.Features.Get()?.Metadata; + Assert.NotNull(metadata); + Assert.Equal("203.0.113.99", metadata!.RemoteIp); + Assert.Equal("203.0.113.99", metadata.ForwardedFor); + } + + private static AuthorityRateLimiterMetadataMiddleware CreateMiddleware() + { + return new AuthorityRateLimiterMetadataMiddleware( + _ => Task.CompletedTask, + TimeProvider.System, + NullLogger.Instance); + } +} diff --git a/src/StellaOps.Authority/StellaOps.Authority.Tests/RateLimiting/AuthorityRateLimiterTests.cs b/src/StellaOps.Authority/StellaOps.Authority.Tests/RateLimiting/AuthorityRateLimiterTests.cs new file mode 100644 index 00000000..53870b25 --- /dev/null +++ b/src/StellaOps.Authority/StellaOps.Authority.Tests/RateLimiting/AuthorityRateLimiterTests.cs @@ -0,0 +1,80 @@ +using System; +using System.Net; +using Microsoft.AspNetCore.Http; +using StellaOps.Authority; +using StellaOps.Configuration; +using Xunit; + +namespace StellaOps.Authority.Tests.RateLimiting; + +public class AuthorityRateLimiterTests +{ + [Fact] + public async Task TokenLimiter_Throttles_WhenLimitExceeded() + { + var options = CreateOptions(); + options.Security.RateLimiting.Token.PermitLimit = 1; + options.Security.RateLimiting.Token.Window = TimeSpan.FromMinutes(1); + + using var limiter = AuthorityRateLimiter.CreateGlobalLimiter(options); + + var context = new DefaultHttpContext(); + context.Request.Path = "/token"; + context.Connection.RemoteIpAddress = IPAddress.Parse("203.0.113.10"); + + var first = await limiter.AcquireAsync(context); + Assert.True(first.IsAcquired); + + var second = await limiter.AcquireAsync(context); + Assert.False(second.IsAcquired); + } + + [Fact] + public async Task AuthorizeLimiter_Allows_WhenDisabled() + { + var options = CreateOptions(); + options.Security.RateLimiting.Authorize.Enabled = false; + + using var limiter = AuthorityRateLimiter.CreateGlobalLimiter(options); + + var context = new DefaultHttpContext(); + context.Request.Path = "/authorize"; + context.Connection.RemoteIpAddress = IPAddress.Parse("203.0.113.20"); + + var lease = await limiter.AcquireAsync(context); + Assert.True(lease.IsAcquired); + } + + [Fact] + public async Task InternalLimiter_Throttles_WhenEnabled() + { + var options = CreateOptions(); + options.Security.RateLimiting.Internal.Enabled = true; + options.Security.RateLimiting.Internal.PermitLimit = 1; + options.Security.RateLimiting.Internal.Window = TimeSpan.FromMinutes(1); + + using var limiter = AuthorityRateLimiter.CreateGlobalLimiter(options); + + var context = new DefaultHttpContext(); + context.Request.Path = "/internal/health"; + context.Connection.RemoteIpAddress = IPAddress.Parse("203.0.113.30"); + + var first = await limiter.AcquireAsync(context); + Assert.True(first.IsAcquired); + + var second = await limiter.AcquireAsync(context); + Assert.False(second.IsAcquired); + } + + private static StellaOpsAuthorityOptions CreateOptions() + { + var options = new StellaOpsAuthorityOptions + { + Issuer = new Uri("https://authority.stella-ops.test"), + SchemaVersion = 1 + }; + + options.Storage.ConnectionString = "mongodb://localhost/authority"; + return options; + } +} diff --git a/src/StellaOps.Authority/StellaOps.Authority.Tests/StellaOps.Authority.Tests.csproj b/src/StellaOps.Authority/StellaOps.Authority.Tests/StellaOps.Authority.Tests.csproj new file mode 100644 index 00000000..ce847ce2 --- /dev/null +++ b/src/StellaOps.Authority/StellaOps.Authority.Tests/StellaOps.Authority.Tests.csproj @@ -0,0 +1,12 @@ + + + net10.0 + enable + enable + false + + + + + + diff --git a/src/StellaOps.Authority/StellaOps.Authority.sln b/src/StellaOps.Authority/StellaOps.Authority.sln new file mode 100644 index 00000000..71a314ab --- /dev/null +++ b/src/StellaOps.Authority/StellaOps.Authority.sln @@ -0,0 +1,370 @@ + +Microsoft Visual Studio Solution File, Format Version 12.00 +# Visual Studio Version 17 +VisualStudioVersion = 17.0.31903.59 +MinimumVisualStudioVersion = 10.0.40219.1 +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "StellaOps.Authority", "StellaOps.Authority\StellaOps.Authority.csproj", "{93CEF308-E217-41F3-BBF3-AFC1D32D9B4C}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "StellaOps.Authority.Plugins.Abstractions", "StellaOps.Authority.Plugins.Abstractions\StellaOps.Authority.Plugins.Abstractions.csproj", "{B4E5DC28-0693-4708-8B07-5206053CACDB}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "StellaOps.Authority.Plugin.Standard", "StellaOps.Authority.Plugin.Standard\StellaOps.Authority.Plugin.Standard.csproj", "{753A4FF4-BE1D-4361-9FE5-F2FF7CBDE3E3}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "StellaOps.Auth.Abstractions", "StellaOps.Auth.Abstractions\StellaOps.Auth.Abstractions.csproj", "{A399A886-B7B7-4ACE-811E-3F4B7051A725}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "StellaOps.Auth.ServerIntegration", "StellaOps.Auth.ServerIntegration\StellaOps.Auth.ServerIntegration.csproj", "{0BA36155-0024-42D9-9DC9-8F85A72F9CA6}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "StellaOps.Auth.Client", "StellaOps.Auth.Client\StellaOps.Auth.Client.csproj", "{9C8918FA-626F-41DE-8B89-4E216DCBF2A8}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "StellaOps.Configuration.Tests", "..\StellaOps.Configuration.Tests\StellaOps.Configuration.Tests.csproj", "{A33529C5-1552-4216-B080-B621F077BE10}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "StellaOps.Plugin", "..\StellaOps.Plugin\StellaOps.Plugin.csproj", "{C8F10390-5ED3-4638-A27E-F53F07583745}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "StellaOps.DependencyInjection", "..\StellaOps.DependencyInjection\StellaOps.DependencyInjection.csproj", "{D3FCB965-348C-4050-B4F7-7E065A562E2C}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "StellaOps.Configuration", "..\StellaOps.Configuration\StellaOps.Configuration.csproj", "{3CB099C3-F41F-46AD-B81D-DB31C4EF643A}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "StellaOps.Authority.Plugins.Abstractions.Tests", "StellaOps.Authority.Plugins.Abstractions.Tests\StellaOps.Authority.Plugins.Abstractions.Tests.csproj", "{EE97137B-22AF-4A84-9F65-9B4C6468B3CF}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "StellaOps.Feedser.Testing", "..\StellaOps.Feedser.Testing\StellaOps.Feedser.Testing.csproj", "{D48E48BF-80C8-43DA-8BE6-E2B9E769C49E}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "StellaOps.Feedser.Source.Common", "..\StellaOps.Feedser.Source.Common\StellaOps.Feedser.Source.Common.csproj", "{E0B9CD7A-C4FF-44EB-BE04-9B998C1C4166}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "StellaOps.Feedser.Storage.Mongo", "..\StellaOps.Feedser.Storage.Mongo\StellaOps.Feedser.Storage.Mongo.csproj", "{67C85AC6-1670-4A0D-A81F-6015574F46C7}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "StellaOps.Feedser.Core", "..\StellaOps.Feedser.Core\StellaOps.Feedser.Core.csproj", "{17829125-C0F5-47E6-A16C-EC142BD58220}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "StellaOps.Feedser.Models", "..\StellaOps.Feedser.Models\StellaOps.Feedser.Models.csproj", "{9B4BA030-C979-4191-8B4F-7E2AD9F88A94}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "StellaOps.Feedser.Normalization", "..\StellaOps.Feedser.Normalization\StellaOps.Feedser.Normalization.csproj", "{26B58A9B-DB0B-4E3D-9827-3722859E5FB4}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "StellaOps.Authority.Tests", "StellaOps.Authority.Tests\StellaOps.Authority.Tests.csproj", "{D719B01C-2424-4DAB-94B9-C9B6004F450B}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "StellaOps.Authority.Plugin.Standard.Tests", "StellaOps.Authority.Plugin.Standard.Tests\StellaOps.Authority.Plugin.Standard.Tests.csproj", "{0C222CD9-96B1-4152-BD29-65FFAE27C880}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "StellaOps.Authority.Storage.Mongo", "StellaOps.Authority.Storage.Mongo\StellaOps.Authority.Storage.Mongo.csproj", "{977FD870-91B5-44BA-944B-496B2C68DAA0}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "StellaOps.Auth.Abstractions.Tests", "StellaOps.Auth.Abstractions.Tests\StellaOps.Auth.Abstractions.Tests.csproj", "{4A5D29B8-959A-4EAC-A827-979CD058EC16}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "StellaOps.Auth.ServerIntegration.Tests", "StellaOps.Auth.ServerIntegration.Tests\StellaOps.Auth.ServerIntegration.Tests.csproj", "{CB7FD547-1EC7-4A6F-87FE-F73003512AFE}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "StellaOps.Auth.Client.Tests", "StellaOps.Auth.Client.Tests\StellaOps.Auth.Client.Tests.csproj", "{2DB48E45-BEFE-40FC-8E7D-1697A8EB0749}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "StellaOps.Cryptography", "..\StellaOps.Cryptography\StellaOps.Cryptography.csproj", "{35D22E43-729A-4D43-A289-5A0E96BA0199}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "StellaOps.Cryptography.Tests", "..\StellaOps.Cryptography.Tests\StellaOps.Cryptography.Tests.csproj", "{84AEC0C8-EE60-4AB1-A59B-B8E7CCFC0A25}" +EndProject +Global + GlobalSection(SolutionConfigurationPlatforms) = preSolution + Debug|Any CPU = Debug|Any CPU + Debug|x64 = Debug|x64 + Debug|x86 = Debug|x86 + Release|Any CPU = Release|Any CPU + Release|x64 = Release|x64 + Release|x86 = Release|x86 + EndGlobalSection + GlobalSection(ProjectConfigurationPlatforms) = postSolution + {93CEF308-E217-41F3-BBF3-AFC1D32D9B4C}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {93CEF308-E217-41F3-BBF3-AFC1D32D9B4C}.Debug|Any CPU.Build.0 = Debug|Any CPU + {93CEF308-E217-41F3-BBF3-AFC1D32D9B4C}.Debug|x64.ActiveCfg = Debug|Any CPU + {93CEF308-E217-41F3-BBF3-AFC1D32D9B4C}.Debug|x64.Build.0 = Debug|Any CPU + {93CEF308-E217-41F3-BBF3-AFC1D32D9B4C}.Debug|x86.ActiveCfg = Debug|Any CPU + {93CEF308-E217-41F3-BBF3-AFC1D32D9B4C}.Debug|x86.Build.0 = Debug|Any CPU + {93CEF308-E217-41F3-BBF3-AFC1D32D9B4C}.Release|Any CPU.ActiveCfg = Release|Any CPU + {93CEF308-E217-41F3-BBF3-AFC1D32D9B4C}.Release|Any CPU.Build.0 = Release|Any CPU + {93CEF308-E217-41F3-BBF3-AFC1D32D9B4C}.Release|x64.ActiveCfg = Release|Any CPU + {93CEF308-E217-41F3-BBF3-AFC1D32D9B4C}.Release|x64.Build.0 = Release|Any CPU + {93CEF308-E217-41F3-BBF3-AFC1D32D9B4C}.Release|x86.ActiveCfg = Release|Any CPU + {93CEF308-E217-41F3-BBF3-AFC1D32D9B4C}.Release|x86.Build.0 = Release|Any CPU + {B4E5DC28-0693-4708-8B07-5206053CACDB}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {B4E5DC28-0693-4708-8B07-5206053CACDB}.Debug|Any CPU.Build.0 = Debug|Any CPU + {B4E5DC28-0693-4708-8B07-5206053CACDB}.Debug|x64.ActiveCfg = Debug|Any CPU + {B4E5DC28-0693-4708-8B07-5206053CACDB}.Debug|x64.Build.0 = Debug|Any CPU + {B4E5DC28-0693-4708-8B07-5206053CACDB}.Debug|x86.ActiveCfg = Debug|Any CPU + {B4E5DC28-0693-4708-8B07-5206053CACDB}.Debug|x86.Build.0 = Debug|Any CPU + {B4E5DC28-0693-4708-8B07-5206053CACDB}.Release|Any CPU.ActiveCfg = Release|Any CPU + {B4E5DC28-0693-4708-8B07-5206053CACDB}.Release|Any CPU.Build.0 = Release|Any CPU + {B4E5DC28-0693-4708-8B07-5206053CACDB}.Release|x64.ActiveCfg = Release|Any CPU + {B4E5DC28-0693-4708-8B07-5206053CACDB}.Release|x64.Build.0 = Release|Any CPU + {B4E5DC28-0693-4708-8B07-5206053CACDB}.Release|x86.ActiveCfg = Release|Any CPU + {B4E5DC28-0693-4708-8B07-5206053CACDB}.Release|x86.Build.0 = Release|Any CPU + {753A4FF4-BE1D-4361-9FE5-F2FF7CBDE3E3}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {753A4FF4-BE1D-4361-9FE5-F2FF7CBDE3E3}.Debug|Any CPU.Build.0 = Debug|Any CPU + {753A4FF4-BE1D-4361-9FE5-F2FF7CBDE3E3}.Debug|x64.ActiveCfg = Debug|Any CPU + {753A4FF4-BE1D-4361-9FE5-F2FF7CBDE3E3}.Debug|x64.Build.0 = Debug|Any CPU + {753A4FF4-BE1D-4361-9FE5-F2FF7CBDE3E3}.Debug|x86.ActiveCfg = Debug|Any CPU + {753A4FF4-BE1D-4361-9FE5-F2FF7CBDE3E3}.Debug|x86.Build.0 = Debug|Any CPU + {753A4FF4-BE1D-4361-9FE5-F2FF7CBDE3E3}.Release|Any CPU.ActiveCfg = Release|Any CPU + {753A4FF4-BE1D-4361-9FE5-F2FF7CBDE3E3}.Release|Any CPU.Build.0 = Release|Any CPU + {753A4FF4-BE1D-4361-9FE5-F2FF7CBDE3E3}.Release|x64.ActiveCfg = Release|Any CPU + {753A4FF4-BE1D-4361-9FE5-F2FF7CBDE3E3}.Release|x64.Build.0 = Release|Any CPU + {753A4FF4-BE1D-4361-9FE5-F2FF7CBDE3E3}.Release|x86.ActiveCfg = Release|Any CPU + {753A4FF4-BE1D-4361-9FE5-F2FF7CBDE3E3}.Release|x86.Build.0 = Release|Any CPU + {A399A886-B7B7-4ACE-811E-3F4B7051A725}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {A399A886-B7B7-4ACE-811E-3F4B7051A725}.Debug|Any CPU.Build.0 = Debug|Any CPU + {A399A886-B7B7-4ACE-811E-3F4B7051A725}.Debug|x64.ActiveCfg = Debug|Any CPU + {A399A886-B7B7-4ACE-811E-3F4B7051A725}.Debug|x64.Build.0 = Debug|Any CPU + {A399A886-B7B7-4ACE-811E-3F4B7051A725}.Debug|x86.ActiveCfg = Debug|Any CPU + {A399A886-B7B7-4ACE-811E-3F4B7051A725}.Debug|x86.Build.0 = Debug|Any CPU + {A399A886-B7B7-4ACE-811E-3F4B7051A725}.Release|Any CPU.ActiveCfg = Release|Any CPU + {A399A886-B7B7-4ACE-811E-3F4B7051A725}.Release|Any CPU.Build.0 = Release|Any CPU + {A399A886-B7B7-4ACE-811E-3F4B7051A725}.Release|x64.ActiveCfg = Release|Any CPU + {A399A886-B7B7-4ACE-811E-3F4B7051A725}.Release|x64.Build.0 = Release|Any CPU + {A399A886-B7B7-4ACE-811E-3F4B7051A725}.Release|x86.ActiveCfg = Release|Any CPU + {A399A886-B7B7-4ACE-811E-3F4B7051A725}.Release|x86.Build.0 = Release|Any CPU + {0BA36155-0024-42D9-9DC9-8F85A72F9CA6}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {0BA36155-0024-42D9-9DC9-8F85A72F9CA6}.Debug|Any CPU.Build.0 = Debug|Any CPU + {0BA36155-0024-42D9-9DC9-8F85A72F9CA6}.Debug|x64.ActiveCfg = Debug|Any CPU + {0BA36155-0024-42D9-9DC9-8F85A72F9CA6}.Debug|x64.Build.0 = Debug|Any CPU + {0BA36155-0024-42D9-9DC9-8F85A72F9CA6}.Debug|x86.ActiveCfg = Debug|Any CPU + {0BA36155-0024-42D9-9DC9-8F85A72F9CA6}.Debug|x86.Build.0 = Debug|Any CPU + {0BA36155-0024-42D9-9DC9-8F85A72F9CA6}.Release|Any CPU.ActiveCfg = Release|Any CPU + {0BA36155-0024-42D9-9DC9-8F85A72F9CA6}.Release|Any CPU.Build.0 = Release|Any CPU + {0BA36155-0024-42D9-9DC9-8F85A72F9CA6}.Release|x64.ActiveCfg = Release|Any CPU + {0BA36155-0024-42D9-9DC9-8F85A72F9CA6}.Release|x64.Build.0 = Release|Any CPU + {0BA36155-0024-42D9-9DC9-8F85A72F9CA6}.Release|x86.ActiveCfg = Release|Any CPU + {0BA36155-0024-42D9-9DC9-8F85A72F9CA6}.Release|x86.Build.0 = Release|Any CPU + {9C8918FA-626F-41DE-8B89-4E216DCBF2A8}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {9C8918FA-626F-41DE-8B89-4E216DCBF2A8}.Debug|Any CPU.Build.0 = Debug|Any CPU + {9C8918FA-626F-41DE-8B89-4E216DCBF2A8}.Debug|x64.ActiveCfg = Debug|Any CPU + {9C8918FA-626F-41DE-8B89-4E216DCBF2A8}.Debug|x64.Build.0 = Debug|Any CPU + {9C8918FA-626F-41DE-8B89-4E216DCBF2A8}.Debug|x86.ActiveCfg = Debug|Any CPU + {9C8918FA-626F-41DE-8B89-4E216DCBF2A8}.Debug|x86.Build.0 = Debug|Any CPU + {9C8918FA-626F-41DE-8B89-4E216DCBF2A8}.Release|Any CPU.ActiveCfg = Release|Any CPU + {9C8918FA-626F-41DE-8B89-4E216DCBF2A8}.Release|Any CPU.Build.0 = Release|Any CPU + {9C8918FA-626F-41DE-8B89-4E216DCBF2A8}.Release|x64.ActiveCfg = Release|Any CPU + {9C8918FA-626F-41DE-8B89-4E216DCBF2A8}.Release|x64.Build.0 = Release|Any CPU + {9C8918FA-626F-41DE-8B89-4E216DCBF2A8}.Release|x86.ActiveCfg = Release|Any CPU + {9C8918FA-626F-41DE-8B89-4E216DCBF2A8}.Release|x86.Build.0 = Release|Any CPU + {A33529C5-1552-4216-B080-B621F077BE10}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {A33529C5-1552-4216-B080-B621F077BE10}.Debug|Any CPU.Build.0 = Debug|Any CPU + {A33529C5-1552-4216-B080-B621F077BE10}.Debug|x64.ActiveCfg = Debug|Any CPU + {A33529C5-1552-4216-B080-B621F077BE10}.Debug|x64.Build.0 = Debug|Any CPU + {A33529C5-1552-4216-B080-B621F077BE10}.Debug|x86.ActiveCfg = Debug|Any CPU + {A33529C5-1552-4216-B080-B621F077BE10}.Debug|x86.Build.0 = Debug|Any CPU + {A33529C5-1552-4216-B080-B621F077BE10}.Release|Any CPU.ActiveCfg = Release|Any CPU + {A33529C5-1552-4216-B080-B621F077BE10}.Release|Any CPU.Build.0 = Release|Any CPU + {A33529C5-1552-4216-B080-B621F077BE10}.Release|x64.ActiveCfg = Release|Any CPU + {A33529C5-1552-4216-B080-B621F077BE10}.Release|x64.Build.0 = Release|Any CPU + {A33529C5-1552-4216-B080-B621F077BE10}.Release|x86.ActiveCfg = Release|Any CPU + {A33529C5-1552-4216-B080-B621F077BE10}.Release|x86.Build.0 = Release|Any CPU + {C8F10390-5ED3-4638-A27E-F53F07583745}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {C8F10390-5ED3-4638-A27E-F53F07583745}.Debug|Any CPU.Build.0 = Debug|Any CPU + {C8F10390-5ED3-4638-A27E-F53F07583745}.Debug|x64.ActiveCfg = Debug|Any CPU + {C8F10390-5ED3-4638-A27E-F53F07583745}.Debug|x64.Build.0 = Debug|Any CPU + {C8F10390-5ED3-4638-A27E-F53F07583745}.Debug|x86.ActiveCfg = Debug|Any CPU + {C8F10390-5ED3-4638-A27E-F53F07583745}.Debug|x86.Build.0 = Debug|Any CPU + {C8F10390-5ED3-4638-A27E-F53F07583745}.Release|Any CPU.ActiveCfg = Release|Any CPU + {C8F10390-5ED3-4638-A27E-F53F07583745}.Release|Any CPU.Build.0 = Release|Any CPU + {C8F10390-5ED3-4638-A27E-F53F07583745}.Release|x64.ActiveCfg = Release|Any CPU + {C8F10390-5ED3-4638-A27E-F53F07583745}.Release|x64.Build.0 = Release|Any CPU + {C8F10390-5ED3-4638-A27E-F53F07583745}.Release|x86.ActiveCfg = Release|Any CPU + {C8F10390-5ED3-4638-A27E-F53F07583745}.Release|x86.Build.0 = Release|Any CPU + {D3FCB965-348C-4050-B4F7-7E065A562E2C}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {D3FCB965-348C-4050-B4F7-7E065A562E2C}.Debug|Any CPU.Build.0 = Debug|Any CPU + {D3FCB965-348C-4050-B4F7-7E065A562E2C}.Debug|x64.ActiveCfg = Debug|Any CPU + {D3FCB965-348C-4050-B4F7-7E065A562E2C}.Debug|x64.Build.0 = Debug|Any CPU + {D3FCB965-348C-4050-B4F7-7E065A562E2C}.Debug|x86.ActiveCfg = Debug|Any CPU + {D3FCB965-348C-4050-B4F7-7E065A562E2C}.Debug|x86.Build.0 = Debug|Any CPU + {D3FCB965-348C-4050-B4F7-7E065A562E2C}.Release|Any CPU.ActiveCfg = Release|Any CPU + {D3FCB965-348C-4050-B4F7-7E065A562E2C}.Release|Any CPU.Build.0 = Release|Any CPU + {D3FCB965-348C-4050-B4F7-7E065A562E2C}.Release|x64.ActiveCfg = Release|Any CPU + {D3FCB965-348C-4050-B4F7-7E065A562E2C}.Release|x64.Build.0 = Release|Any CPU + {D3FCB965-348C-4050-B4F7-7E065A562E2C}.Release|x86.ActiveCfg = Release|Any CPU + {D3FCB965-348C-4050-B4F7-7E065A562E2C}.Release|x86.Build.0 = Release|Any CPU + {3CB099C3-F41F-46AD-B81D-DB31C4EF643A}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {3CB099C3-F41F-46AD-B81D-DB31C4EF643A}.Debug|Any CPU.Build.0 = Debug|Any CPU + {3CB099C3-F41F-46AD-B81D-DB31C4EF643A}.Debug|x64.ActiveCfg = Debug|Any CPU + {3CB099C3-F41F-46AD-B81D-DB31C4EF643A}.Debug|x64.Build.0 = Debug|Any CPU + {3CB099C3-F41F-46AD-B81D-DB31C4EF643A}.Debug|x86.ActiveCfg = Debug|Any CPU + {3CB099C3-F41F-46AD-B81D-DB31C4EF643A}.Debug|x86.Build.0 = Debug|Any CPU + {3CB099C3-F41F-46AD-B81D-DB31C4EF643A}.Release|Any CPU.ActiveCfg = Release|Any CPU + {3CB099C3-F41F-46AD-B81D-DB31C4EF643A}.Release|Any CPU.Build.0 = Release|Any CPU + {3CB099C3-F41F-46AD-B81D-DB31C4EF643A}.Release|x64.ActiveCfg = Release|Any CPU + {3CB099C3-F41F-46AD-B81D-DB31C4EF643A}.Release|x64.Build.0 = Release|Any CPU + {3CB099C3-F41F-46AD-B81D-DB31C4EF643A}.Release|x86.ActiveCfg = Release|Any CPU + {3CB099C3-F41F-46AD-B81D-DB31C4EF643A}.Release|x86.Build.0 = Release|Any CPU + {EE97137B-22AF-4A84-9F65-9B4C6468B3CF}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {EE97137B-22AF-4A84-9F65-9B4C6468B3CF}.Debug|Any CPU.Build.0 = Debug|Any CPU + {EE97137B-22AF-4A84-9F65-9B4C6468B3CF}.Debug|x64.ActiveCfg = Debug|Any CPU + {EE97137B-22AF-4A84-9F65-9B4C6468B3CF}.Debug|x64.Build.0 = Debug|Any CPU + {EE97137B-22AF-4A84-9F65-9B4C6468B3CF}.Debug|x86.ActiveCfg = Debug|Any CPU + {EE97137B-22AF-4A84-9F65-9B4C6468B3CF}.Debug|x86.Build.0 = Debug|Any CPU + {EE97137B-22AF-4A84-9F65-9B4C6468B3CF}.Release|Any CPU.ActiveCfg = Release|Any CPU + {EE97137B-22AF-4A84-9F65-9B4C6468B3CF}.Release|Any CPU.Build.0 = Release|Any CPU + {EE97137B-22AF-4A84-9F65-9B4C6468B3CF}.Release|x64.ActiveCfg = Release|Any CPU + {EE97137B-22AF-4A84-9F65-9B4C6468B3CF}.Release|x64.Build.0 = Release|Any CPU + {EE97137B-22AF-4A84-9F65-9B4C6468B3CF}.Release|x86.ActiveCfg = Release|Any CPU + {EE97137B-22AF-4A84-9F65-9B4C6468B3CF}.Release|x86.Build.0 = Release|Any CPU + {D48E48BF-80C8-43DA-8BE6-E2B9E769C49E}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {D48E48BF-80C8-43DA-8BE6-E2B9E769C49E}.Debug|Any CPU.Build.0 = Debug|Any CPU + {D48E48BF-80C8-43DA-8BE6-E2B9E769C49E}.Debug|x64.ActiveCfg = Debug|Any CPU + {D48E48BF-80C8-43DA-8BE6-E2B9E769C49E}.Debug|x64.Build.0 = Debug|Any CPU + {D48E48BF-80C8-43DA-8BE6-E2B9E769C49E}.Debug|x86.ActiveCfg = Debug|Any CPU + {D48E48BF-80C8-43DA-8BE6-E2B9E769C49E}.Debug|x86.Build.0 = Debug|Any CPU + {D48E48BF-80C8-43DA-8BE6-E2B9E769C49E}.Release|Any CPU.ActiveCfg = Release|Any CPU + {D48E48BF-80C8-43DA-8BE6-E2B9E769C49E}.Release|Any CPU.Build.0 = Release|Any CPU + {D48E48BF-80C8-43DA-8BE6-E2B9E769C49E}.Release|x64.ActiveCfg = Release|Any CPU + {D48E48BF-80C8-43DA-8BE6-E2B9E769C49E}.Release|x64.Build.0 = Release|Any CPU + {D48E48BF-80C8-43DA-8BE6-E2B9E769C49E}.Release|x86.ActiveCfg = Release|Any CPU + {D48E48BF-80C8-43DA-8BE6-E2B9E769C49E}.Release|x86.Build.0 = Release|Any CPU + {E0B9CD7A-C4FF-44EB-BE04-9B998C1C4166}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {E0B9CD7A-C4FF-44EB-BE04-9B998C1C4166}.Debug|Any CPU.Build.0 = Debug|Any CPU + {E0B9CD7A-C4FF-44EB-BE04-9B998C1C4166}.Debug|x64.ActiveCfg = Debug|Any CPU + {E0B9CD7A-C4FF-44EB-BE04-9B998C1C4166}.Debug|x64.Build.0 = Debug|Any CPU + {E0B9CD7A-C4FF-44EB-BE04-9B998C1C4166}.Debug|x86.ActiveCfg = Debug|Any CPU + {E0B9CD7A-C4FF-44EB-BE04-9B998C1C4166}.Debug|x86.Build.0 = Debug|Any CPU + {E0B9CD7A-C4FF-44EB-BE04-9B998C1C4166}.Release|Any CPU.ActiveCfg = Release|Any CPU + {E0B9CD7A-C4FF-44EB-BE04-9B998C1C4166}.Release|Any CPU.Build.0 = Release|Any CPU + {E0B9CD7A-C4FF-44EB-BE04-9B998C1C4166}.Release|x64.ActiveCfg = Release|Any CPU + {E0B9CD7A-C4FF-44EB-BE04-9B998C1C4166}.Release|x64.Build.0 = Release|Any CPU + {E0B9CD7A-C4FF-44EB-BE04-9B998C1C4166}.Release|x86.ActiveCfg = Release|Any CPU + {E0B9CD7A-C4FF-44EB-BE04-9B998C1C4166}.Release|x86.Build.0 = Release|Any CPU + {67C85AC6-1670-4A0D-A81F-6015574F46C7}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {67C85AC6-1670-4A0D-A81F-6015574F46C7}.Debug|Any CPU.Build.0 = Debug|Any CPU + {67C85AC6-1670-4A0D-A81F-6015574F46C7}.Debug|x64.ActiveCfg = Debug|Any CPU + {67C85AC6-1670-4A0D-A81F-6015574F46C7}.Debug|x64.Build.0 = Debug|Any CPU + {67C85AC6-1670-4A0D-A81F-6015574F46C7}.Debug|x86.ActiveCfg = Debug|Any CPU + {67C85AC6-1670-4A0D-A81F-6015574F46C7}.Debug|x86.Build.0 = Debug|Any CPU + {67C85AC6-1670-4A0D-A81F-6015574F46C7}.Release|Any CPU.ActiveCfg = Release|Any CPU + {67C85AC6-1670-4A0D-A81F-6015574F46C7}.Release|Any CPU.Build.0 = Release|Any CPU + {67C85AC6-1670-4A0D-A81F-6015574F46C7}.Release|x64.ActiveCfg = Release|Any CPU + {67C85AC6-1670-4A0D-A81F-6015574F46C7}.Release|x64.Build.0 = Release|Any CPU + {67C85AC6-1670-4A0D-A81F-6015574F46C7}.Release|x86.ActiveCfg = Release|Any CPU + {67C85AC6-1670-4A0D-A81F-6015574F46C7}.Release|x86.Build.0 = Release|Any CPU + {17829125-C0F5-47E6-A16C-EC142BD58220}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {17829125-C0F5-47E6-A16C-EC142BD58220}.Debug|Any CPU.Build.0 = Debug|Any CPU + {17829125-C0F5-47E6-A16C-EC142BD58220}.Debug|x64.ActiveCfg = Debug|Any CPU + {17829125-C0F5-47E6-A16C-EC142BD58220}.Debug|x64.Build.0 = Debug|Any CPU + {17829125-C0F5-47E6-A16C-EC142BD58220}.Debug|x86.ActiveCfg = Debug|Any CPU + {17829125-C0F5-47E6-A16C-EC142BD58220}.Debug|x86.Build.0 = Debug|Any CPU + {17829125-C0F5-47E6-A16C-EC142BD58220}.Release|Any CPU.ActiveCfg = Release|Any CPU + {17829125-C0F5-47E6-A16C-EC142BD58220}.Release|Any CPU.Build.0 = Release|Any CPU + {17829125-C0F5-47E6-A16C-EC142BD58220}.Release|x64.ActiveCfg = Release|Any CPU + {17829125-C0F5-47E6-A16C-EC142BD58220}.Release|x64.Build.0 = Release|Any CPU + {17829125-C0F5-47E6-A16C-EC142BD58220}.Release|x86.ActiveCfg = Release|Any CPU + {17829125-C0F5-47E6-A16C-EC142BD58220}.Release|x86.Build.0 = Release|Any CPU + {9B4BA030-C979-4191-8B4F-7E2AD9F88A94}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {9B4BA030-C979-4191-8B4F-7E2AD9F88A94}.Debug|Any CPU.Build.0 = Debug|Any CPU + {9B4BA030-C979-4191-8B4F-7E2AD9F88A94}.Debug|x64.ActiveCfg = Debug|Any CPU + {9B4BA030-C979-4191-8B4F-7E2AD9F88A94}.Debug|x64.Build.0 = Debug|Any CPU + {9B4BA030-C979-4191-8B4F-7E2AD9F88A94}.Debug|x86.ActiveCfg = Debug|Any CPU + {9B4BA030-C979-4191-8B4F-7E2AD9F88A94}.Debug|x86.Build.0 = Debug|Any CPU + {9B4BA030-C979-4191-8B4F-7E2AD9F88A94}.Release|Any CPU.ActiveCfg = Release|Any CPU + {9B4BA030-C979-4191-8B4F-7E2AD9F88A94}.Release|Any CPU.Build.0 = Release|Any CPU + {9B4BA030-C979-4191-8B4F-7E2AD9F88A94}.Release|x64.ActiveCfg = Release|Any CPU + {9B4BA030-C979-4191-8B4F-7E2AD9F88A94}.Release|x64.Build.0 = Release|Any CPU + {9B4BA030-C979-4191-8B4F-7E2AD9F88A94}.Release|x86.ActiveCfg = Release|Any CPU + {9B4BA030-C979-4191-8B4F-7E2AD9F88A94}.Release|x86.Build.0 = Release|Any CPU + {26B58A9B-DB0B-4E3D-9827-3722859E5FB4}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {26B58A9B-DB0B-4E3D-9827-3722859E5FB4}.Debug|Any CPU.Build.0 = Debug|Any CPU + {26B58A9B-DB0B-4E3D-9827-3722859E5FB4}.Debug|x64.ActiveCfg = Debug|Any CPU + {26B58A9B-DB0B-4E3D-9827-3722859E5FB4}.Debug|x64.Build.0 = Debug|Any CPU + {26B58A9B-DB0B-4E3D-9827-3722859E5FB4}.Debug|x86.ActiveCfg = Debug|Any CPU + {26B58A9B-DB0B-4E3D-9827-3722859E5FB4}.Debug|x86.Build.0 = Debug|Any CPU + {26B58A9B-DB0B-4E3D-9827-3722859E5FB4}.Release|Any CPU.ActiveCfg = Release|Any CPU + {26B58A9B-DB0B-4E3D-9827-3722859E5FB4}.Release|Any CPU.Build.0 = Release|Any CPU + {26B58A9B-DB0B-4E3D-9827-3722859E5FB4}.Release|x64.ActiveCfg = Release|Any CPU + {26B58A9B-DB0B-4E3D-9827-3722859E5FB4}.Release|x64.Build.0 = Release|Any CPU + {26B58A9B-DB0B-4E3D-9827-3722859E5FB4}.Release|x86.ActiveCfg = Release|Any CPU + {26B58A9B-DB0B-4E3D-9827-3722859E5FB4}.Release|x86.Build.0 = Release|Any CPU + {D719B01C-2424-4DAB-94B9-C9B6004F450B}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {D719B01C-2424-4DAB-94B9-C9B6004F450B}.Debug|Any CPU.Build.0 = Debug|Any CPU + {D719B01C-2424-4DAB-94B9-C9B6004F450B}.Debug|x64.ActiveCfg = Debug|Any CPU + {D719B01C-2424-4DAB-94B9-C9B6004F450B}.Debug|x64.Build.0 = Debug|Any CPU + {D719B01C-2424-4DAB-94B9-C9B6004F450B}.Debug|x86.ActiveCfg = Debug|Any CPU + {D719B01C-2424-4DAB-94B9-C9B6004F450B}.Debug|x86.Build.0 = Debug|Any CPU + {D719B01C-2424-4DAB-94B9-C9B6004F450B}.Release|Any CPU.ActiveCfg = Release|Any CPU + {D719B01C-2424-4DAB-94B9-C9B6004F450B}.Release|Any CPU.Build.0 = Release|Any CPU + {D719B01C-2424-4DAB-94B9-C9B6004F450B}.Release|x64.ActiveCfg = Release|Any CPU + {D719B01C-2424-4DAB-94B9-C9B6004F450B}.Release|x64.Build.0 = Release|Any CPU + {D719B01C-2424-4DAB-94B9-C9B6004F450B}.Release|x86.ActiveCfg = Release|Any CPU + {D719B01C-2424-4DAB-94B9-C9B6004F450B}.Release|x86.Build.0 = Release|Any CPU + {0C222CD9-96B1-4152-BD29-65FFAE27C880}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {0C222CD9-96B1-4152-BD29-65FFAE27C880}.Debug|Any CPU.Build.0 = Debug|Any CPU + {0C222CD9-96B1-4152-BD29-65FFAE27C880}.Debug|x64.ActiveCfg = Debug|Any CPU + {0C222CD9-96B1-4152-BD29-65FFAE27C880}.Debug|x64.Build.0 = Debug|Any CPU + {0C222CD9-96B1-4152-BD29-65FFAE27C880}.Debug|x86.ActiveCfg = Debug|Any CPU + {0C222CD9-96B1-4152-BD29-65FFAE27C880}.Debug|x86.Build.0 = Debug|Any CPU + {0C222CD9-96B1-4152-BD29-65FFAE27C880}.Release|Any CPU.ActiveCfg = Release|Any CPU + {0C222CD9-96B1-4152-BD29-65FFAE27C880}.Release|Any CPU.Build.0 = Release|Any CPU + {0C222CD9-96B1-4152-BD29-65FFAE27C880}.Release|x64.ActiveCfg = Release|Any CPU + {0C222CD9-96B1-4152-BD29-65FFAE27C880}.Release|x64.Build.0 = Release|Any CPU + {0C222CD9-96B1-4152-BD29-65FFAE27C880}.Release|x86.ActiveCfg = Release|Any CPU + {0C222CD9-96B1-4152-BD29-65FFAE27C880}.Release|x86.Build.0 = Release|Any CPU + {977FD870-91B5-44BA-944B-496B2C68DAA0}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {977FD870-91B5-44BA-944B-496B2C68DAA0}.Debug|Any CPU.Build.0 = Debug|Any CPU + {977FD870-91B5-44BA-944B-496B2C68DAA0}.Debug|x64.ActiveCfg = Debug|Any CPU + {977FD870-91B5-44BA-944B-496B2C68DAA0}.Debug|x64.Build.0 = Debug|Any CPU + {977FD870-91B5-44BA-944B-496B2C68DAA0}.Debug|x86.ActiveCfg = Debug|Any CPU + {977FD870-91B5-44BA-944B-496B2C68DAA0}.Debug|x86.Build.0 = Debug|Any CPU + {977FD870-91B5-44BA-944B-496B2C68DAA0}.Release|Any CPU.ActiveCfg = Release|Any CPU + {977FD870-91B5-44BA-944B-496B2C68DAA0}.Release|Any CPU.Build.0 = Release|Any CPU + {977FD870-91B5-44BA-944B-496B2C68DAA0}.Release|x64.ActiveCfg = Release|Any CPU + {977FD870-91B5-44BA-944B-496B2C68DAA0}.Release|x64.Build.0 = Release|Any CPU + {977FD870-91B5-44BA-944B-496B2C68DAA0}.Release|x86.ActiveCfg = Release|Any CPU + {977FD870-91B5-44BA-944B-496B2C68DAA0}.Release|x86.Build.0 = Release|Any CPU + {4A5D29B8-959A-4EAC-A827-979CD058EC16}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {4A5D29B8-959A-4EAC-A827-979CD058EC16}.Debug|Any CPU.Build.0 = Debug|Any CPU + {4A5D29B8-959A-4EAC-A827-979CD058EC16}.Debug|x64.ActiveCfg = Debug|Any CPU + {4A5D29B8-959A-4EAC-A827-979CD058EC16}.Debug|x64.Build.0 = Debug|Any CPU + {4A5D29B8-959A-4EAC-A827-979CD058EC16}.Debug|x86.ActiveCfg = Debug|Any CPU + {4A5D29B8-959A-4EAC-A827-979CD058EC16}.Debug|x86.Build.0 = Debug|Any CPU + {4A5D29B8-959A-4EAC-A827-979CD058EC16}.Release|Any CPU.ActiveCfg = Release|Any CPU + {4A5D29B8-959A-4EAC-A827-979CD058EC16}.Release|Any CPU.Build.0 = Release|Any CPU + {4A5D29B8-959A-4EAC-A827-979CD058EC16}.Release|x64.ActiveCfg = Release|Any CPU + {4A5D29B8-959A-4EAC-A827-979CD058EC16}.Release|x64.Build.0 = Release|Any CPU + {4A5D29B8-959A-4EAC-A827-979CD058EC16}.Release|x86.ActiveCfg = Release|Any CPU + {4A5D29B8-959A-4EAC-A827-979CD058EC16}.Release|x86.Build.0 = Release|Any CPU + {CB7FD547-1EC7-4A6F-87FE-F73003512AFE}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {CB7FD547-1EC7-4A6F-87FE-F73003512AFE}.Debug|Any CPU.Build.0 = Debug|Any CPU + {CB7FD547-1EC7-4A6F-87FE-F73003512AFE}.Debug|x64.ActiveCfg = Debug|Any CPU + {CB7FD547-1EC7-4A6F-87FE-F73003512AFE}.Debug|x64.Build.0 = Debug|Any CPU + {CB7FD547-1EC7-4A6F-87FE-F73003512AFE}.Debug|x86.ActiveCfg = Debug|Any CPU + {CB7FD547-1EC7-4A6F-87FE-F73003512AFE}.Debug|x86.Build.0 = Debug|Any CPU + {CB7FD547-1EC7-4A6F-87FE-F73003512AFE}.Release|Any CPU.ActiveCfg = Release|Any CPU + {CB7FD547-1EC7-4A6F-87FE-F73003512AFE}.Release|Any CPU.Build.0 = Release|Any CPU + {CB7FD547-1EC7-4A6F-87FE-F73003512AFE}.Release|x64.ActiveCfg = Release|Any CPU + {CB7FD547-1EC7-4A6F-87FE-F73003512AFE}.Release|x64.Build.0 = Release|Any CPU + {CB7FD547-1EC7-4A6F-87FE-F73003512AFE}.Release|x86.ActiveCfg = Release|Any CPU + {CB7FD547-1EC7-4A6F-87FE-F73003512AFE}.Release|x86.Build.0 = Release|Any CPU + {2DB48E45-BEFE-40FC-8E7D-1697A8EB0749}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {2DB48E45-BEFE-40FC-8E7D-1697A8EB0749}.Debug|Any CPU.Build.0 = Debug|Any CPU + {2DB48E45-BEFE-40FC-8E7D-1697A8EB0749}.Debug|x64.ActiveCfg = Debug|Any CPU + {2DB48E45-BEFE-40FC-8E7D-1697A8EB0749}.Debug|x64.Build.0 = Debug|Any CPU + {2DB48E45-BEFE-40FC-8E7D-1697A8EB0749}.Debug|x86.ActiveCfg = Debug|Any CPU + {2DB48E45-BEFE-40FC-8E7D-1697A8EB0749}.Debug|x86.Build.0 = Debug|Any CPU + {2DB48E45-BEFE-40FC-8E7D-1697A8EB0749}.Release|Any CPU.ActiveCfg = Release|Any CPU + {2DB48E45-BEFE-40FC-8E7D-1697A8EB0749}.Release|Any CPU.Build.0 = Release|Any CPU + {2DB48E45-BEFE-40FC-8E7D-1697A8EB0749}.Release|x64.ActiveCfg = Release|Any CPU + {2DB48E45-BEFE-40FC-8E7D-1697A8EB0749}.Release|x64.Build.0 = Release|Any CPU + {2DB48E45-BEFE-40FC-8E7D-1697A8EB0749}.Release|x86.ActiveCfg = Release|Any CPU + {2DB48E45-BEFE-40FC-8E7D-1697A8EB0749}.Release|x86.Build.0 = Release|Any CPU + {35D22E43-729A-4D43-A289-5A0E96BA0199}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {35D22E43-729A-4D43-A289-5A0E96BA0199}.Debug|Any CPU.Build.0 = Debug|Any CPU + {35D22E43-729A-4D43-A289-5A0E96BA0199}.Debug|x64.ActiveCfg = Debug|Any CPU + {35D22E43-729A-4D43-A289-5A0E96BA0199}.Debug|x64.Build.0 = Debug|Any CPU + {35D22E43-729A-4D43-A289-5A0E96BA0199}.Debug|x86.ActiveCfg = Debug|Any CPU + {35D22E43-729A-4D43-A289-5A0E96BA0199}.Debug|x86.Build.0 = Debug|Any CPU + {35D22E43-729A-4D43-A289-5A0E96BA0199}.Release|Any CPU.ActiveCfg = Release|Any CPU + {35D22E43-729A-4D43-A289-5A0E96BA0199}.Release|Any CPU.Build.0 = Release|Any CPU + {35D22E43-729A-4D43-A289-5A0E96BA0199}.Release|x64.ActiveCfg = Release|Any CPU + {35D22E43-729A-4D43-A289-5A0E96BA0199}.Release|x64.Build.0 = Release|Any CPU + {35D22E43-729A-4D43-A289-5A0E96BA0199}.Release|x86.ActiveCfg = Release|Any CPU + {35D22E43-729A-4D43-A289-5A0E96BA0199}.Release|x86.Build.0 = Release|Any CPU + {84AEC0C8-EE60-4AB1-A59B-B8E7CCFC0A25}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {84AEC0C8-EE60-4AB1-A59B-B8E7CCFC0A25}.Debug|Any CPU.Build.0 = Debug|Any CPU + {84AEC0C8-EE60-4AB1-A59B-B8E7CCFC0A25}.Debug|x64.ActiveCfg = Debug|Any CPU + {84AEC0C8-EE60-4AB1-A59B-B8E7CCFC0A25}.Debug|x64.Build.0 = Debug|Any CPU + {84AEC0C8-EE60-4AB1-A59B-B8E7CCFC0A25}.Debug|x86.ActiveCfg = Debug|Any CPU + {84AEC0C8-EE60-4AB1-A59B-B8E7CCFC0A25}.Debug|x86.Build.0 = Debug|Any CPU + {84AEC0C8-EE60-4AB1-A59B-B8E7CCFC0A25}.Release|Any CPU.ActiveCfg = Release|Any CPU + {84AEC0C8-EE60-4AB1-A59B-B8E7CCFC0A25}.Release|Any CPU.Build.0 = Release|Any CPU + {84AEC0C8-EE60-4AB1-A59B-B8E7CCFC0A25}.Release|x64.ActiveCfg = Release|Any CPU + {84AEC0C8-EE60-4AB1-A59B-B8E7CCFC0A25}.Release|x64.Build.0 = Release|Any CPU + {84AEC0C8-EE60-4AB1-A59B-B8E7CCFC0A25}.Release|x86.ActiveCfg = Release|Any CPU + {84AEC0C8-EE60-4AB1-A59B-B8E7CCFC0A25}.Release|x86.Build.0 = Release|Any CPU + EndGlobalSection + GlobalSection(SolutionProperties) = preSolution + HideSolutionNode = FALSE + EndGlobalSection +EndGlobal diff --git a/src/StellaOps.Authority/StellaOps.Authority/AuthorityIdentityProviderRegistry.cs b/src/StellaOps.Authority/StellaOps.Authority/AuthorityIdentityProviderRegistry.cs new file mode 100644 index 00000000..f1289283 --- /dev/null +++ b/src/StellaOps.Authority/StellaOps.Authority/AuthorityIdentityProviderRegistry.cs @@ -0,0 +1,103 @@ +using System.Collections.ObjectModel; +using System.Diagnostics.CodeAnalysis; +using Microsoft.Extensions.Logging; +using StellaOps.Authority.Plugins.Abstractions; + +namespace StellaOps.Authority; + +internal sealed class AuthorityIdentityProviderRegistry : IAuthorityIdentityProviderRegistry +{ + private readonly IReadOnlyDictionary providersByName; + private readonly ReadOnlyCollection providers; + private readonly ReadOnlyCollection passwordProviders; + private readonly ReadOnlyCollection mfaProviders; + private readonly ReadOnlyCollection clientProvisioningProviders; + + public AuthorityIdentityProviderRegistry( + IEnumerable providerInstances, + ILogger logger) + { + logger = logger ?? throw new ArgumentNullException(nameof(logger)); + + var orderedProviders = providerInstances? + .Where(static p => p is not null) + .OrderBy(static p => p.Name, StringComparer.OrdinalIgnoreCase) + .ToList() ?? new List(); + + var uniqueProviders = new List(orderedProviders.Count); + var password = new List(); + var mfa = new List(); + var clientProvisioning = new List(); + + var dictionary = new Dictionary(StringComparer.OrdinalIgnoreCase); + + foreach (var provider in orderedProviders) + { + if (string.IsNullOrWhiteSpace(provider.Name)) + { + logger.LogWarning( + "Identity provider plugin of type '{PluginType}' was registered with an empty name and will be ignored.", + provider.Type); + continue; + } + + if (!dictionary.TryAdd(provider.Name, provider)) + { + logger.LogWarning( + "Duplicate identity provider name '{PluginName}' detected; ignoring additional registration for type '{PluginType}'.", + provider.Name, + provider.Type); + continue; + } + + uniqueProviders.Add(provider); + + if (provider.Capabilities.SupportsPassword) + { + password.Add(provider); + } + + if (provider.Capabilities.SupportsMfa) + { + mfa.Add(provider); + } + + if (provider.Capabilities.SupportsClientProvisioning) + { + clientProvisioning.Add(provider); + } + } + + providersByName = dictionary; + providers = new ReadOnlyCollection(uniqueProviders); + passwordProviders = new ReadOnlyCollection(password); + mfaProviders = new ReadOnlyCollection(mfa); + clientProvisioningProviders = new ReadOnlyCollection(clientProvisioning); + + AggregateCapabilities = new AuthorityIdentityProviderCapabilities( + SupportsPassword: passwordProviders.Count > 0, + SupportsMfa: mfaProviders.Count > 0, + SupportsClientProvisioning: clientProvisioningProviders.Count > 0); + } + + public IReadOnlyCollection Providers => providers; + + public IReadOnlyCollection PasswordProviders => passwordProviders; + + public IReadOnlyCollection MfaProviders => mfaProviders; + + public IReadOnlyCollection ClientProvisioningProviders => clientProvisioningProviders; + + public AuthorityIdentityProviderCapabilities AggregateCapabilities { get; } + + public bool TryGet(string name, [NotNullWhen(true)] out IIdentityProviderPlugin? provider) + { + if (string.IsNullOrWhiteSpace(name)) + { + provider = null; + return false; + } + + return providersByName.TryGetValue(name, out provider); + } +} diff --git a/src/StellaOps.Authority/StellaOps.Authority/AuthorityPluginRegistry.cs b/src/StellaOps.Authority/StellaOps.Authority/AuthorityPluginRegistry.cs new file mode 100644 index 00000000..5ba56e34 --- /dev/null +++ b/src/StellaOps.Authority/StellaOps.Authority/AuthorityPluginRegistry.cs @@ -0,0 +1,23 @@ +using System; +using System.Collections.Generic; +using System.Diagnostics.CodeAnalysis; +using System.Linq; +using StellaOps.Authority.Plugins.Abstractions; + +namespace StellaOps.Authority; + +internal sealed class AuthorityPluginRegistry : IAuthorityPluginRegistry +{ + private readonly IReadOnlyDictionary registry; + + public AuthorityPluginRegistry(IEnumerable contexts) + { + registry = contexts.ToDictionary(c => c.Manifest.Name, StringComparer.OrdinalIgnoreCase); + Plugins = registry.Values.ToArray(); + } + + public IReadOnlyCollection Plugins { get; } + + public bool TryGet(string name, [NotNullWhen(true)] out AuthorityPluginContext? context) + => registry.TryGetValue(name, out context); +} diff --git a/src/StellaOps.Authority/StellaOps.Authority/AuthorityRateLimiter.cs b/src/StellaOps.Authority/StellaOps.Authority/AuthorityRateLimiter.cs new file mode 100644 index 00000000..11f67938 --- /dev/null +++ b/src/StellaOps.Authority/StellaOps.Authority/AuthorityRateLimiter.cs @@ -0,0 +1,117 @@ +using System; +using System.Globalization; +using System.Net; +using System.Threading.RateLimiting; +using Microsoft.AspNetCore.Http; +using Microsoft.AspNetCore.RateLimiting; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Logging; +using StellaOps.Authority.RateLimiting; +using StellaOps.Configuration; + +namespace StellaOps.Authority; + +internal static class AuthorityRateLimiter +{ + internal const string TokenLimiterName = "authority-token"; + internal const string AuthorizeLimiterName = "authority-authorize"; + internal const string InternalLimiterName = "authority-internal"; + private static readonly IAuthorityRateLimiterPartitionKeyResolver FallbackPartitionKeyResolver = new DefaultAuthorityRateLimiterPartitionKeyResolver(); + + public static void Configure(RateLimiterOptions options, StellaOpsAuthorityOptions authorityOptions, ILogger? logger = null) + { + ArgumentNullException.ThrowIfNull(options); + ArgumentNullException.ThrowIfNull(authorityOptions); + + options.RejectionStatusCode = StatusCodes.Status429TooManyRequests; + options.GlobalLimiter = CreateGlobalLimiter(authorityOptions); + options.OnRejected = async (context, cancellationToken) => + { + var httpContext = context.HttpContext; + var remoteIp = httpContext.Connection.RemoteIpAddress?.ToString() ?? "unknown"; + logger ??= httpContext.RequestServices.GetService(typeof(ILoggerFactory)) is ILoggerFactory loggerFactory + ? loggerFactory.CreateLogger("StellaOps.Authority.RateLimiting") + : null; + + logger?.LogWarning( + "Rate limit exceeded for path {Path} from {RemoteIp}.", + httpContext.Request.Path, + remoteIp); + + if (context.Lease.TryGetMetadata(MetadataName.RetryAfter, out TimeSpan retryAfter)) + { + httpContext.Response.Headers["Retry-After"] = Math.Ceiling(retryAfter.TotalSeconds).ToString(CultureInfo.InvariantCulture); + } + + await ValueTask.CompletedTask; + }; + } + + public static PartitionedRateLimiter CreateGlobalLimiter(StellaOpsAuthorityOptions authorityOptions) + { + ArgumentNullException.ThrowIfNull(authorityOptions); + + var rateLimiting = authorityOptions.Security.RateLimiting; + var tokenOptions = rateLimiting.Token; + var authorizeOptions = rateLimiting.Authorize; + var internalOptions = rateLimiting.Internal; + + return PartitionedRateLimiter.Create(context => + { + var path = context.Request.Path; + + if (tokenOptions.Enabled && path.HasValue && path.Value!.Equals("/token", StringComparison.OrdinalIgnoreCase)) + { + var partitionKey = ResolvePartitionKey(context, TokenLimiterName); + return RateLimitPartition.GetFixedWindowLimiter( + partitionKey, + _ => CreateFixedWindowOptions(tokenOptions)); + } + + if (authorizeOptions.Enabled && path.HasValue && path.Value!.Equals("/authorize", StringComparison.OrdinalIgnoreCase)) + { + var partitionKey = ResolvePartitionKey(context, AuthorizeLimiterName); + return RateLimitPartition.GetFixedWindowLimiter( + partitionKey, + _ => CreateFixedWindowOptions(authorizeOptions)); + } + + if (internalOptions.Enabled && path.HasValue && path.Value!.StartsWith("/internal", StringComparison.OrdinalIgnoreCase)) + { + var partitionKey = ResolvePartitionKey(context, InternalLimiterName); + return RateLimitPartition.GetFixedWindowLimiter( + partitionKey, + _ => CreateFixedWindowOptions(internalOptions)); + } + + return RateLimitPartition.GetNoLimiter(ResolvePartitionKey(context)); + }); + } + + private static FixedWindowRateLimiterOptions CreateFixedWindowOptions(AuthorityEndpointRateLimitOptions options) + { + return new FixedWindowRateLimiterOptions + { + PermitLimit = options.PermitLimit, + QueueLimit = options.QueueLimit, + QueueProcessingOrder = options.QueueProcessingOrder, + Window = options.Window, + AutoReplenishment = true + }; + } + + private static string ResolvePartitionKey(HttpContext context, string limiterName) + { + var metadata = context.Features.Get()?.Metadata ?? new AuthorityRateLimiterMetadata(); + var serviceProvider = context.RequestServices; + var resolver = serviceProvider is not null + ? serviceProvider.GetService(typeof(IAuthorityRateLimiterPartitionKeyResolver)) as IAuthorityRateLimiterPartitionKeyResolver ?? FallbackPartitionKeyResolver + : FallbackPartitionKeyResolver; + return resolver.Resolve(limiterName, context, metadata); + } + + private static string ResolvePartitionKey(HttpContext context) + { + return ResolvePartitionKey(context, "none"); + } +} diff --git a/src/StellaOps.Authority/StellaOps.Authority/AuthorityTelemetryConfiguration.cs b/src/StellaOps.Authority/StellaOps.Authority/AuthorityTelemetryConfiguration.cs new file mode 100644 index 00000000..ad59d398 --- /dev/null +++ b/src/StellaOps.Authority/StellaOps.Authority/AuthorityTelemetryConfiguration.cs @@ -0,0 +1,60 @@ +using System; +using System.Collections.Generic; +using System.Diagnostics; +using System.Diagnostics.Metrics; +using Microsoft.AspNetCore.Builder; +using Microsoft.Extensions.DependencyInjection; +using OpenTelemetry.Extensions.Hosting; +using OpenTelemetry.Metrics; +using OpenTelemetry.Resources; +using OpenTelemetry.Trace; +using StellaOps.Auth; + +namespace StellaOps.Authority; + +/// +/// Configures OpenTelemetry primitives for the Authority host. +/// +internal static class AuthorityTelemetryConfiguration +{ + public static void Configure(WebApplicationBuilder builder) + { + ArgumentNullException.ThrowIfNull(builder); + + builder.Services.AddSingleton(_ => new ActivitySource(AuthorityTelemetry.ActivitySourceName)); + builder.Services.AddSingleton(_ => new Meter(AuthorityTelemetry.MeterName)); + + var openTelemetry = builder.Services.AddOpenTelemetry(); + var serviceVersion = AuthorityTelemetry.ResolveServiceVersion(typeof(AuthorityTelemetryConfiguration).Assembly); + + openTelemetry.ConfigureResource(resource => + { + resource.AddService( + serviceName: AuthorityTelemetry.ServiceName, + serviceNamespace: AuthorityTelemetry.ServiceNamespace, + serviceVersion: serviceVersion, + serviceInstanceId: Environment.MachineName); + + resource.AddAttributes(new[] + { + new KeyValuePair("deployment.environment", builder.Environment.EnvironmentName) + }); + }); + + openTelemetry.WithTracing(tracing => + { + tracing + .AddSource(AuthorityTelemetry.ActivitySourceName) + .AddAspNetCoreInstrumentation() + .AddHttpClientInstrumentation(); + }); + + openTelemetry.WithMetrics(metrics => + { + metrics + .AddMeter(AuthorityTelemetry.MeterName) + .AddAspNetCoreInstrumentation() + .AddRuntimeInstrumentation(); + }); + } +} diff --git a/src/StellaOps.Authority/StellaOps.Authority/Bootstrap/BootstrapApiKeyFilter.cs b/src/StellaOps.Authority/StellaOps.Authority/Bootstrap/BootstrapApiKeyFilter.cs new file mode 100644 index 00000000..c15311e6 --- /dev/null +++ b/src/StellaOps.Authority/StellaOps.Authority/Bootstrap/BootstrapApiKeyFilter.cs @@ -0,0 +1,30 @@ +using Microsoft.AspNetCore.Http; +using StellaOps.Configuration; + +namespace StellaOps.Authority.Bootstrap; + +internal sealed class BootstrapApiKeyFilter : IEndpointFilter +{ + private readonly StellaOpsAuthorityOptions options; + + public BootstrapApiKeyFilter(StellaOpsAuthorityOptions options) + { + this.options = options ?? throw new ArgumentNullException(nameof(options)); + } + + public async ValueTask InvokeAsync(EndpointFilterInvocationContext context, EndpointFilterDelegate next) + { + if (!options.Bootstrap.Enabled) + { + return Results.NotFound(); + } + + if (!context.HttpContext.Request.Headers.TryGetValue("X-StellaOps-Bootstrap-Key", out var key) || + !string.Equals(key.ToString(), options.Bootstrap.ApiKey, StringComparison.Ordinal)) + { + return Results.Unauthorized(); + } + + return await next(context).ConfigureAwait(false); + } +} diff --git a/src/StellaOps.Authority/StellaOps.Authority/Bootstrap/BootstrapRequests.cs b/src/StellaOps.Authority/StellaOps.Authority/Bootstrap/BootstrapRequests.cs new file mode 100644 index 00000000..f5c31955 --- /dev/null +++ b/src/StellaOps.Authority/StellaOps.Authority/Bootstrap/BootstrapRequests.cs @@ -0,0 +1,48 @@ +using System.ComponentModel.DataAnnotations; + +namespace StellaOps.Authority.Bootstrap; + +internal sealed record BootstrapUserRequest +{ + public string? Provider { get; init; } + + [Required] + public string Username { get; init; } = string.Empty; + + [Required] + public string Password { get; init; } = string.Empty; + + public string? DisplayName { get; init; } + + public string? Email { get; init; } + + public bool RequirePasswordReset { get; init; } + + public IReadOnlyCollection? Roles { get; init; } + + public IReadOnlyDictionary? Attributes { get; init; } +} + +internal sealed record BootstrapClientRequest +{ + public string? Provider { get; init; } + + [Required] + public string ClientId { get; init; } = string.Empty; + + public bool Confidential { get; init; } = true; + + public string? DisplayName { get; init; } + + public string? ClientSecret { get; init; } + + public IReadOnlyCollection? AllowedGrantTypes { get; init; } + + public IReadOnlyCollection? AllowedScopes { get; init; } + + public IReadOnlyCollection? RedirectUris { get; init; } + + public IReadOnlyCollection? PostLogoutRedirectUris { get; init; } + + public IReadOnlyDictionary? Properties { get; init; } +} diff --git a/src/StellaOps.Authority/StellaOps.Authority/OpenIddict/AuthorityIdentityProviderSelector.cs b/src/StellaOps.Authority/StellaOps.Authority/OpenIddict/AuthorityIdentityProviderSelector.cs new file mode 100644 index 00000000..95091a6f --- /dev/null +++ b/src/StellaOps.Authority/StellaOps.Authority/OpenIddict/AuthorityIdentityProviderSelector.cs @@ -0,0 +1,63 @@ +using OpenIddict.Abstractions; +using StellaOps.Authority.Plugins.Abstractions; + +namespace StellaOps.Authority.OpenIddict; + +internal static class AuthorityIdentityProviderSelector +{ + public static ProviderSelectionResult ResolvePasswordProvider(OpenIddictRequest request, IAuthorityIdentityProviderRegistry registry) + { + ArgumentNullException.ThrowIfNull(request); + ArgumentNullException.ThrowIfNull(registry); + + if (registry.PasswordProviders.Count == 0) + { + return ProviderSelectionResult.Failure( + OpenIddictConstants.Errors.UnsupportedGrantType, + "Password grants are not enabled because no identity providers support password authentication."); + } + + var providerName = request.GetParameter(AuthorityOpenIddictConstants.ProviderParameterName)?.Value?.ToString(); + if (string.IsNullOrWhiteSpace(providerName)) + { + if (registry.PasswordProviders.Count == 1) + { + var provider = registry.PasswordProviders.First(); + return ProviderSelectionResult.Success(provider); + } + + return ProviderSelectionResult.Failure( + OpenIddictConstants.Errors.InvalidRequest, + "identity_provider parameter is required when multiple password-capable providers are registered."); + } + + if (!registry.TryGet(providerName!, out var selected)) + { + return ProviderSelectionResult.Failure( + OpenIddictConstants.Errors.InvalidRequest, + $"Unknown identity provider '{providerName}'."); + } + + if (!selected.Capabilities.SupportsPassword) + { + return ProviderSelectionResult.Failure( + OpenIddictConstants.Errors.InvalidRequest, + $"Identity provider '{providerName}' does not support password authentication."); + } + + return ProviderSelectionResult.Success(selected); + } + + internal sealed record ProviderSelectionResult( + bool Succeeded, + IIdentityProviderPlugin? Provider, + string? Error, + string? Description) + { + public static ProviderSelectionResult Success(IIdentityProviderPlugin provider) + => new(true, provider, null, null); + + public static ProviderSelectionResult Failure(string error, string description) + => new(false, null, error, description); + } +} diff --git a/src/StellaOps.Authority/StellaOps.Authority/OpenIddict/AuthorityOpenIddictConstants.cs b/src/StellaOps.Authority/StellaOps.Authority/OpenIddict/AuthorityOpenIddictConstants.cs new file mode 100644 index 00000000..868aa5e1 --- /dev/null +++ b/src/StellaOps.Authority/StellaOps.Authority/OpenIddict/AuthorityOpenIddictConstants.cs @@ -0,0 +1,11 @@ +namespace StellaOps.Authority.OpenIddict; + +internal static class AuthorityOpenIddictConstants +{ + internal const string ProviderParameterName = "authority_provider"; + internal const string ProviderTransactionProperty = "authority:identity_provider"; + internal const string ClientTransactionProperty = "authority:client"; + internal const string ClientProviderTransactionProperty = "authority:client_provider"; + internal const string ClientGrantedScopesProperty = "authority:client_granted_scopes"; + internal const string TokenTransactionProperty = "authority:token"; +} diff --git a/src/StellaOps.Authority/StellaOps.Authority/OpenIddict/Handlers/ClientCredentialsHandlers.cs b/src/StellaOps.Authority/StellaOps.Authority/OpenIddict/Handlers/ClientCredentialsHandlers.cs new file mode 100644 index 00000000..26942cb2 --- /dev/null +++ b/src/StellaOps.Authority/StellaOps.Authority/OpenIddict/Handlers/ClientCredentialsHandlers.cs @@ -0,0 +1,394 @@ +using System.Diagnostics; +using System.Linq; +using System.Security.Claims; +using System.Security.Cryptography; +using Microsoft.Extensions.Logging; +using OpenIddict.Abstractions; +using OpenIddict.Extensions; +using OpenIddict.Server; +using OpenIddict.Server.AspNetCore; +using StellaOps.Auth.Abstractions; +using StellaOps.Authority.OpenIddict; +using StellaOps.Authority.Plugins.Abstractions; +using StellaOps.Authority.Storage.Mongo.Documents; +using StellaOps.Authority.Storage.Mongo.Stores; + +namespace StellaOps.Authority.OpenIddict.Handlers; + +internal sealed class ValidateClientCredentialsHandler : IOpenIddictServerHandler +{ + private readonly IAuthorityClientStore clientStore; + private readonly IAuthorityIdentityProviderRegistry registry; + private readonly ActivitySource activitySource; + private readonly ILogger logger; + + public ValidateClientCredentialsHandler( + IAuthorityClientStore clientStore, + IAuthorityIdentityProviderRegistry registry, + ActivitySource activitySource, + ILogger logger) + { + this.clientStore = clientStore ?? throw new ArgumentNullException(nameof(clientStore)); + this.registry = registry ?? throw new ArgumentNullException(nameof(registry)); + this.activitySource = activitySource ?? throw new ArgumentNullException(nameof(activitySource)); + this.logger = logger ?? throw new ArgumentNullException(nameof(logger)); + } + + public async ValueTask HandleAsync(OpenIddictServerEvents.ValidateTokenRequestContext context) + { + ArgumentNullException.ThrowIfNull(context); + + if (!context.Request.IsClientCredentialsGrantType()) + { + return; + } + + using var activity = activitySource.StartActivity("authority.token.validate_client_credentials", ActivityKind.Internal); + activity?.SetTag("authority.endpoint", "/token"); + activity?.SetTag("authority.grant_type", OpenIddictConstants.GrantTypes.ClientCredentials); + activity?.SetTag("authority.client_id", context.ClientId ?? string.Empty); + + if (string.IsNullOrWhiteSpace(context.ClientId)) + { + context.Reject(OpenIddictConstants.Errors.InvalidClient, "Client identifier is required."); + logger.LogWarning("Client credentials validation failed: missing client identifier."); + return; + } + + var document = await clientStore.FindByClientIdAsync(context.ClientId, context.CancellationToken).ConfigureAwait(false); + if (document is null || document.Disabled) + { + context.Reject(OpenIddictConstants.Errors.InvalidClient, "Unknown or disabled client identifier."); + logger.LogWarning("Client credentials validation failed for {ClientId}: client not found or disabled.", context.ClientId); + return; + } + + IIdentityProviderPlugin? provider = null; + if (!string.IsNullOrWhiteSpace(document.Plugin)) + { + if (!registry.TryGet(document.Plugin, out provider)) + { + context.Reject(OpenIddictConstants.Errors.InvalidClient, "Configured identity provider is unavailable."); + logger.LogWarning("Client credentials validation failed for {ClientId}: provider {Provider} unavailable.", context.ClientId, document.Plugin); + return; + } + + if (!provider.Capabilities.SupportsClientProvisioning || provider.ClientProvisioning is null) + { + context.Reject(OpenIddictConstants.Errors.UnauthorizedClient, "Associated identity provider does not support client provisioning."); + logger.LogWarning("Client credentials validation failed for {ClientId}: provider {Provider} lacks client provisioning capabilities.", context.ClientId, provider.Name); + return; + } + } + + var allowedGrantTypes = ClientCredentialHandlerHelpers.Split(document.Properties, AuthorityClientMetadataKeys.AllowedGrantTypes); + if (allowedGrantTypes.Count > 0 && + !allowedGrantTypes.Any(static grant => string.Equals(grant, OpenIddictConstants.GrantTypes.ClientCredentials, StringComparison.Ordinal))) + { + context.Reject(OpenIddictConstants.Errors.UnauthorizedClient, "Client credentials grant is not permitted for this client."); + logger.LogWarning("Client credentials validation failed for {ClientId}: grant type not allowed.", document.ClientId); + return; + } + + var requiresSecret = string.Equals(document.ClientType, "confidential", StringComparison.OrdinalIgnoreCase); + if (requiresSecret) + { + if (string.IsNullOrWhiteSpace(document.SecretHash)) + { + context.Reject(OpenIddictConstants.Errors.InvalidClient, "Client secret is not configured."); + logger.LogWarning("Client credentials validation failed for {ClientId}: secret not configured.", document.ClientId); + return; + } + + if (string.IsNullOrWhiteSpace(context.ClientSecret) || + !ClientCredentialHandlerHelpers.VerifySecret(context.ClientSecret, document.SecretHash)) + { + context.Reject(OpenIddictConstants.Errors.InvalidClient, "Invalid client credentials."); + logger.LogWarning("Client credentials validation failed for {ClientId}: secret verification failed.", document.ClientId); + return; + } + } + else if (!string.IsNullOrWhiteSpace(context.ClientSecret) && !string.IsNullOrWhiteSpace(document.SecretHash) && + !ClientCredentialHandlerHelpers.VerifySecret(context.ClientSecret, document.SecretHash)) + { + context.Reject(OpenIddictConstants.Errors.InvalidClient, "Invalid client credentials."); + logger.LogWarning("Client credentials validation failed for {ClientId}: secret verification failed.", document.ClientId); + return; + } + + var allowedScopes = ClientCredentialHandlerHelpers.Split(document.Properties, AuthorityClientMetadataKeys.AllowedScopes); + var resolvedScopes = ClientCredentialHandlerHelpers.ResolveGrantedScopes( + allowedScopes, + context.Request.GetScopes()); + + if (resolvedScopes.InvalidScope is not null) + { + context.Reject(OpenIddictConstants.Errors.InvalidScope, $"Scope '{resolvedScopes.InvalidScope}' is not allowed for this client."); + logger.LogWarning("Client credentials validation failed for {ClientId}: scope {Scope} not permitted.", document.ClientId, resolvedScopes.InvalidScope); + return; + } + + context.Transaction.Properties[AuthorityOpenIddictConstants.ClientTransactionProperty] = document; + if (provider is not null) + { + context.Transaction.Properties[AuthorityOpenIddictConstants.ClientProviderTransactionProperty] = provider.Name; + activity?.SetTag("authority.identity_provider", provider.Name); + } + + context.Transaction.Properties[AuthorityOpenIddictConstants.ClientGrantedScopesProperty] = resolvedScopes.Scopes; + logger.LogInformation("Client credentials validated for {ClientId}.", document.ClientId); + } +} + +internal sealed class HandleClientCredentialsHandler : IOpenIddictServerHandler +{ + private readonly IAuthorityIdentityProviderRegistry registry; + private readonly IAuthorityTokenStore tokenStore; + private readonly TimeProvider clock; + private readonly ActivitySource activitySource; + private readonly ILogger logger; + + public HandleClientCredentialsHandler( + IAuthorityIdentityProviderRegistry registry, + IAuthorityTokenStore tokenStore, + TimeProvider clock, + ActivitySource activitySource, + ILogger logger) + { + this.registry = registry ?? throw new ArgumentNullException(nameof(registry)); + this.tokenStore = tokenStore ?? throw new ArgumentNullException(nameof(tokenStore)); + this.clock = clock ?? throw new ArgumentNullException(nameof(clock)); + this.activitySource = activitySource ?? throw new ArgumentNullException(nameof(activitySource)); + this.logger = logger ?? throw new ArgumentNullException(nameof(logger)); + } + + public async ValueTask HandleAsync(OpenIddictServerEvents.HandleTokenRequestContext context) + { + ArgumentNullException.ThrowIfNull(context); + + if (!context.Request.IsClientCredentialsGrantType()) + { + return; + } + + using var activity = activitySource.StartActivity("authority.token.handle_client_credentials", ActivityKind.Internal); + + if (!context.Transaction.Properties.TryGetValue(AuthorityOpenIddictConstants.ClientTransactionProperty, out var value) || + value is not AuthorityClientDocument document) + { + context.Reject(OpenIddictConstants.Errors.ServerError, "Client metadata not available."); + return; + } + + var identity = new ClaimsIdentity(OpenIddictServerAspNetCoreDefaults.AuthenticationScheme); + identity.AddClaim(new Claim(OpenIddictConstants.Claims.Subject, document.ClientId)); + identity.AddClaim(new Claim(OpenIddictConstants.Claims.ClientId, document.ClientId)); + activity?.SetTag("authority.client_id", document.ClientId); + activity?.SetTag("authority.endpoint", "/token"); + activity?.SetTag("authority.grant_type", OpenIddictConstants.GrantTypes.ClientCredentials); + + var tokenId = identity.GetClaim(OpenIddictConstants.Claims.JwtId); + if (string.IsNullOrEmpty(tokenId)) + { + tokenId = Guid.NewGuid().ToString("N"); + identity.SetClaim(OpenIddictConstants.Claims.JwtId, tokenId); + } + + identity.SetDestinations(static claim => claim.Type switch + { + OpenIddictConstants.Claims.Subject => new[] { OpenIddictConstants.Destinations.AccessToken }, + OpenIddictConstants.Claims.ClientId => new[] { OpenIddictConstants.Destinations.AccessToken }, + OpenIddictConstants.Claims.JwtId => new[] { OpenIddictConstants.Destinations.AccessToken }, + StellaOpsClaimTypes.IdentityProvider => new[] { OpenIddictConstants.Destinations.AccessToken }, + _ => new[] { OpenIddictConstants.Destinations.AccessToken } + }); + + var (provider, descriptor) = await ResolveProviderAsync(context, document).ConfigureAwait(false); + if (context.IsRejected) + { + logger.LogWarning("Client credentials request rejected for {ClientId} during provider resolution.", document.ClientId); + return; + } + + if (provider is null) + { + if (!string.IsNullOrWhiteSpace(document.Plugin)) + { + identity.SetClaim(StellaOpsClaimTypes.IdentityProvider, document.Plugin); + activity?.SetTag("authority.identity_provider", document.Plugin); + } + } + else + { + identity.SetClaim(StellaOpsClaimTypes.IdentityProvider, provider.Name); + activity?.SetTag("authority.identity_provider", provider.Name); + } + + var principal = new ClaimsPrincipal(identity); + + var grantedScopes = context.Transaction.Properties.TryGetValue(AuthorityOpenIddictConstants.ClientGrantedScopesProperty, out var scopesValue) && + scopesValue is IReadOnlyList resolvedScopes + ? resolvedScopes + : ClientCredentialHandlerHelpers.Split(document.Properties, AuthorityClientMetadataKeys.AllowedScopes); + + if (grantedScopes.Count > 0) + { + principal.SetScopes(grantedScopes); + } + else + { + principal.SetScopes(Array.Empty()); + } + + if (provider is not null && descriptor is not null) + { + var enrichmentContext = new AuthorityClaimsEnrichmentContext(provider.Context, user: null, descriptor); + await provider.ClaimsEnricher.EnrichAsync(identity, enrichmentContext, context.CancellationToken).ConfigureAwait(false); + } + + await PersistTokenAsync(context, document, tokenId, grantedScopes, activity).ConfigureAwait(false); + + context.Principal = principal; + context.HandleRequest(); + logger.LogInformation("Issued client credentials access token for {ClientId} with scopes {Scopes}.", document.ClientId, grantedScopes); + } + + private async ValueTask<(IIdentityProviderPlugin? Provider, AuthorityClientDescriptor? Descriptor)> ResolveProviderAsync( + OpenIddictServerEvents.HandleTokenRequestContext context, + AuthorityClientDocument document) + { + string? providerName = null; + if (context.Transaction.Properties.TryGetValue(AuthorityOpenIddictConstants.ClientProviderTransactionProperty, out var providerValue) && + providerValue is string storedProvider) + { + providerName = storedProvider; + } + else if (!string.IsNullOrWhiteSpace(document.Plugin)) + { + providerName = document.Plugin; + } + + if (string.IsNullOrWhiteSpace(providerName)) + { + return (null, null); + } + + if (!registry.TryGet(providerName, out var provider) || provider.ClientProvisioning is null) + { + context.Reject(OpenIddictConstants.Errors.InvalidClient, "Configured identity provider is unavailable."); + return (null, null); + } + + var descriptor = await provider.ClientProvisioning.FindByClientIdAsync(document.ClientId, context.CancellationToken).ConfigureAwait(false); + + if (descriptor is null) + { + context.Reject(OpenIddictConstants.Errors.InvalidClient, "Client registration was not found."); + return (null, null); + } + + return (provider, descriptor); + } + + private async ValueTask PersistTokenAsync( + OpenIddictServerEvents.HandleTokenRequestContext context, + AuthorityClientDocument document, + string tokenId, + IReadOnlyCollection scopes, + Activity? activity) + { + if (context.IsRejected) + { + return; + } + + var issuedAt = clock.GetUtcNow(); + var lifetime = context.Options?.AccessTokenLifetime; + var expiresAt = lifetime.HasValue && lifetime.Value > TimeSpan.Zero + ? issuedAt + lifetime.Value + : (DateTimeOffset?)null; + + var record = new AuthorityTokenDocument + { + TokenId = tokenId, + Type = OpenIddictConstants.TokenTypeHints.AccessToken, + SubjectId = document.ClientId, + ClientId = document.ClientId, + Scope = scopes.Count > 0 ? scopes.ToList() : new List(), + Status = "valid", + CreatedAt = issuedAt, + ExpiresAt = expiresAt + }; + + await tokenStore.InsertAsync(record, context.CancellationToken).ConfigureAwait(false); + context.Transaction.Properties[AuthorityOpenIddictConstants.TokenTransactionProperty] = record; + activity?.SetTag("authority.token_id", tokenId); + } +} + +internal static class ClientCredentialHandlerHelpers +{ + public static IReadOnlyList Split(IReadOnlyDictionary properties, string key) + { + if (!properties.TryGetValue(key, out var value) || string.IsNullOrWhiteSpace(value)) + { + return Array.Empty(); + } + + return value.Split(' ', StringSplitOptions.RemoveEmptyEntries | StringSplitOptions.TrimEntries); + } + + public static (string[] Scopes, string? InvalidScope) ResolveGrantedScopes( + IReadOnlyCollection allowedScopes, + IReadOnlyList requestedScopes) + { + if (allowedScopes.Count == 0) + { + return (requestedScopes.Count == 0 ? Array.Empty() : requestedScopes.ToArray(), null); + } + + var allowed = new HashSet(allowedScopes, StringComparer.Ordinal); + + if (requestedScopes.Count == 0) + { + return (allowedScopes.ToArray(), null); + } + + foreach (var scope in requestedScopes) + { + if (!allowed.Contains(scope)) + { + return (Array.Empty(), scope); + } + } + + return (requestedScopes.ToArray(), null); + } + + public static bool VerifySecret(string secret, string storedHash) + { + ArgumentException.ThrowIfNullOrWhiteSpace(secret); + + if (string.IsNullOrWhiteSpace(storedHash)) + { + return false; + } + + try + { + var computed = Convert.FromBase64String(AuthoritySecretHasher.ComputeHash(secret)); + var expected = Convert.FromBase64String(storedHash); + + if (computed.Length != expected.Length) + { + return false; + } + + return CryptographicOperations.FixedTimeEquals(computed, expected); + } + catch (FormatException) + { + return false; + } + } +} diff --git a/src/StellaOps.Authority/StellaOps.Authority/OpenIddict/Handlers/PasswordGrantHandlers.cs b/src/StellaOps.Authority/StellaOps.Authority/OpenIddict/Handlers/PasswordGrantHandlers.cs new file mode 100644 index 00000000..09f55fc1 --- /dev/null +++ b/src/StellaOps.Authority/StellaOps.Authority/OpenIddict/Handlers/PasswordGrantHandlers.cs @@ -0,0 +1,187 @@ +using System.Diagnostics; +using System.Security.Claims; +using Microsoft.Extensions.Logging; +using OpenIddict.Abstractions; +using OpenIddict.Extensions; +using OpenIddict.Server; +using OpenIddict.Server.AspNetCore; +using StellaOps.Authority.OpenIddict; +using StellaOps.Authority.Plugins.Abstractions; + +namespace StellaOps.Authority.OpenIddict.Handlers; + +internal sealed class ValidatePasswordGrantHandler : IOpenIddictServerHandler +{ + private readonly IAuthorityIdentityProviderRegistry registry; + private readonly ActivitySource activitySource; + private readonly ILogger logger; + + public ValidatePasswordGrantHandler( + IAuthorityIdentityProviderRegistry registry, + ActivitySource activitySource, + ILogger logger) + { + this.registry = registry ?? throw new ArgumentNullException(nameof(registry)); + this.activitySource = activitySource ?? throw new ArgumentNullException(nameof(activitySource)); + this.logger = logger ?? throw new ArgumentNullException(nameof(logger)); + } + + public ValueTask HandleAsync(OpenIddictServerEvents.ValidateTokenRequestContext context) + { + ArgumentNullException.ThrowIfNull(context); + + if (!context.Request.IsPasswordGrantType()) + { + return default; + } + + using var activity = activitySource.StartActivity("authority.token.validate_password_grant", ActivityKind.Internal); + activity?.SetTag("authority.endpoint", "/token"); + activity?.SetTag("authority.grant_type", OpenIddictConstants.GrantTypes.Password); + activity?.SetTag("authority.username", context.Request.Username ?? string.Empty); + + var selection = AuthorityIdentityProviderSelector.ResolvePasswordProvider(context.Request, registry); + if (!selection.Succeeded) + { + context.Reject(selection.Error!, selection.Description); + logger.LogWarning("Password grant validation failed for {Username}: {Reason}.", context.Request.Username, selection.Description); + return default; + } + + if (string.IsNullOrWhiteSpace(context.Request.Username) || string.IsNullOrEmpty(context.Request.Password)) + { + context.Reject(OpenIddictConstants.Errors.InvalidRequest, "Both username and password must be provided."); + logger.LogWarning("Password grant validation failed: missing credentials for {Username}.", context.Request.Username); + return default; + } + + context.Transaction.Properties[AuthorityOpenIddictConstants.ProviderTransactionProperty] = selection.Provider!.Name; + activity?.SetTag("authority.identity_provider", selection.Provider.Name); + logger.LogInformation("Password grant validation succeeded for {Username} using provider {Provider}.", context.Request.Username, selection.Provider.Name); + return default; + } +} + +internal sealed class HandlePasswordGrantHandler : IOpenIddictServerHandler +{ + private readonly IAuthorityIdentityProviderRegistry registry; + private readonly ActivitySource activitySource; + private readonly ILogger logger; + + public HandlePasswordGrantHandler( + IAuthorityIdentityProviderRegistry registry, + ActivitySource activitySource, + ILogger logger) + { + this.registry = registry ?? throw new ArgumentNullException(nameof(registry)); + this.activitySource = activitySource ?? throw new ArgumentNullException(nameof(activitySource)); + this.logger = logger ?? throw new ArgumentNullException(nameof(logger)); + } + + public async ValueTask HandleAsync(OpenIddictServerEvents.HandleTokenRequestContext context) + { + ArgumentNullException.ThrowIfNull(context); + + if (!context.Request.IsPasswordGrantType()) + { + return; + } + + using var activity = activitySource.StartActivity("authority.token.handle_password_grant", ActivityKind.Internal); + activity?.SetTag("authority.endpoint", "/token"); + activity?.SetTag("authority.grant_type", OpenIddictConstants.GrantTypes.Password); + activity?.SetTag("authority.username", context.Request.Username ?? string.Empty); + + var providerName = context.Transaction.Properties.TryGetValue(AuthorityOpenIddictConstants.ProviderTransactionProperty, out var value) + ? value as string + : null; + + IIdentityProviderPlugin? resolvedProvider; + if (!string.IsNullOrWhiteSpace(providerName)) + { + if (!registry.TryGet(providerName!, out var explicitProvider)) + { + context.Reject(OpenIddictConstants.Errors.ServerError, "Unable to resolve the requested identity provider."); + logger.LogError("Password grant handling failed: provider {Provider} not found for user {Username}.", providerName, context.Request.Username); + return; + } + + resolvedProvider = explicitProvider; + } + else + { + var selection = AuthorityIdentityProviderSelector.ResolvePasswordProvider(context.Request, registry); + if (!selection.Succeeded) + { + context.Reject(selection.Error!, selection.Description); + logger.LogWarning("Password grant handling rejected {Username}: {Reason}.", context.Request.Username, selection.Description); + return; + } + + resolvedProvider = selection.Provider; + } + + var provider = resolvedProvider ?? throw new InvalidOperationException("No identity provider resolved for password grant."); + + var username = context.Request.Username; + var password = context.Request.Password; + if (string.IsNullOrWhiteSpace(username) || string.IsNullOrEmpty(password)) + { + context.Reject(OpenIddictConstants.Errors.InvalidRequest, "Both username and password must be provided."); + logger.LogWarning("Password grant handling rejected: missing credentials for {Username}.", username); + return; + } + + var verification = await provider.Credentials.VerifyPasswordAsync( + username, + password, + context.CancellationToken).ConfigureAwait(false); + + if (!verification.Succeeded || verification.User is null) + { + context.Reject( + OpenIddictConstants.Errors.InvalidGrant, + verification.Message ?? "Invalid username or password."); + logger.LogWarning("Password verification failed for {Username}: {Message}.", username, verification.Message); + return; + } + + var identity = new ClaimsIdentity( + OpenIddictServerAspNetCoreDefaults.AuthenticationScheme, + OpenIddictConstants.Claims.Name, + OpenIddictConstants.Claims.Role); + + identity.AddClaim(new Claim(OpenIddictConstants.Claims.Subject, verification.User.SubjectId)); + identity.AddClaim(new Claim(OpenIddictConstants.Claims.PreferredUsername, verification.User.Username)); + + if (!string.IsNullOrWhiteSpace(verification.User.DisplayName)) + { + identity.AddClaim(new Claim(OpenIddictConstants.Claims.Name, verification.User.DisplayName!)); + } + + foreach (var role in verification.User.Roles) + { + identity.AddClaim(new Claim(OpenIddictConstants.Claims.Role, role)); + } + + identity.SetDestinations(static claim => claim.Type switch + { + OpenIddictConstants.Claims.Subject => new[] { OpenIddictConstants.Destinations.AccessToken, OpenIddictConstants.Destinations.IdentityToken }, + OpenIddictConstants.Claims.Name => new[] { OpenIddictConstants.Destinations.AccessToken, OpenIddictConstants.Destinations.IdentityToken }, + OpenIddictConstants.Claims.PreferredUsername => new[] { OpenIddictConstants.Destinations.AccessToken }, + OpenIddictConstants.Claims.Role => new[] { OpenIddictConstants.Destinations.AccessToken }, + _ => new[] { OpenIddictConstants.Destinations.AccessToken } + }); + + var principal = new ClaimsPrincipal(identity); + principal.SetScopes(context.Request.GetScopes()); + + var enrichmentContext = new AuthorityClaimsEnrichmentContext(provider.Context, verification.User, null); + await provider.ClaimsEnricher.EnrichAsync(identity, enrichmentContext, context.CancellationToken).ConfigureAwait(false); + + context.Principal = principal; + context.HandleRequest(); + activity?.SetTag("authority.subject_id", verification.User.SubjectId); + logger.LogInformation("Password grant issued for {Username} with subject {SubjectId}.", verification.User.Username, verification.User.SubjectId); + } +} diff --git a/src/StellaOps.Authority/StellaOps.Authority/OpenIddict/Handlers/TokenValidationHandlers.cs b/src/StellaOps.Authority/StellaOps.Authority/OpenIddict/Handlers/TokenValidationHandlers.cs new file mode 100644 index 00000000..4a9ab81a --- /dev/null +++ b/src/StellaOps.Authority/StellaOps.Authority/OpenIddict/Handlers/TokenValidationHandlers.cs @@ -0,0 +1,147 @@ +using System.Diagnostics; +using System.Security.Claims; +using Microsoft.Extensions.Logging; +using OpenIddict.Abstractions; +using OpenIddict.Extensions; +using OpenIddict.Server; +using StellaOps.Auth.Abstractions; +using StellaOps.Authority.OpenIddict; +using StellaOps.Authority.Plugins.Abstractions; +using StellaOps.Authority.Storage.Mongo.Documents; +using StellaOps.Authority.Storage.Mongo.Stores; + +namespace StellaOps.Authority.OpenIddict.Handlers; + +internal sealed class ValidateAccessTokenHandler : IOpenIddictServerHandler +{ + private readonly IAuthorityTokenStore tokenStore; + private readonly IAuthorityClientStore clientStore; + private readonly IAuthorityIdentityProviderRegistry registry; + private readonly TimeProvider clock; + private readonly ActivitySource activitySource; + private readonly ILogger logger; + + public ValidateAccessTokenHandler( + IAuthorityTokenStore tokenStore, + IAuthorityClientStore clientStore, + IAuthorityIdentityProviderRegistry registry, + TimeProvider clock, + ActivitySource activitySource, + ILogger logger) + { + this.tokenStore = tokenStore ?? throw new ArgumentNullException(nameof(tokenStore)); + this.clientStore = clientStore ?? throw new ArgumentNullException(nameof(clientStore)); + this.registry = registry ?? throw new ArgumentNullException(nameof(registry)); + this.clock = clock ?? throw new ArgumentNullException(nameof(clock)); + this.activitySource = activitySource ?? throw new ArgumentNullException(nameof(activitySource)); + this.logger = logger ?? throw new ArgumentNullException(nameof(logger)); + } + + public async ValueTask HandleAsync(OpenIddictServerEvents.ValidateTokenContext context) + { + ArgumentNullException.ThrowIfNull(context); + + if (context.Principal is null) + { + return; + } + + if (context.EndpointType is not (OpenIddictServerEndpointType.Token or OpenIddictServerEndpointType.Introspection)) + { + return; + } + + using var activity = activitySource.StartActivity("authority.token.validate_access", ActivityKind.Internal); + activity?.SetTag("authority.endpoint", context.EndpointType switch + { + OpenIddictServerEndpointType.Token => "/token", + OpenIddictServerEndpointType.Introspection => "/introspect", + _ => context.EndpointType.ToString() + }); + + var tokenId = !string.IsNullOrWhiteSpace(context.TokenId) + ? context.TokenId + : context.Principal.GetClaim(OpenIddictConstants.Claims.JwtId); + + if (!string.IsNullOrWhiteSpace(tokenId)) + { + var tokenDocument = await tokenStore.FindByTokenIdAsync(tokenId, context.CancellationToken).ConfigureAwait(false); + if (tokenDocument is not null) + { + if (!string.Equals(tokenDocument.Status, "valid", StringComparison.OrdinalIgnoreCase)) + { + context.Reject(OpenIddictConstants.Errors.InvalidToken, "The token is no longer active."); + logger.LogWarning("Access token {TokenId} rejected: status {Status}.", tokenId, tokenDocument.Status); + return; + } + + if (tokenDocument.ExpiresAt is { } expiresAt && expiresAt <= clock.GetUtcNow()) + { + context.Reject(OpenIddictConstants.Errors.InvalidToken, "The token has expired."); + logger.LogWarning("Access token {TokenId} rejected: expired at {ExpiresAt:o}.", tokenId, expiresAt); + return; + } + + context.Transaction.Properties[AuthorityOpenIddictConstants.TokenTransactionProperty] = tokenDocument; + activity?.SetTag("authority.token_id", tokenDocument.TokenId); + } + } + + var clientId = context.Principal.GetClaim(OpenIddictConstants.Claims.ClientId); + if (!string.IsNullOrWhiteSpace(clientId)) + { + var clientDocument = await clientStore.FindByClientIdAsync(clientId, context.CancellationToken).ConfigureAwait(false); + if (clientDocument is null || clientDocument.Disabled) + { + context.Reject(OpenIddictConstants.Errors.InvalidClient, "The client associated with the token is not permitted."); + logger.LogWarning("Access token validation failed: client {ClientId} disabled or missing.", clientId); + return; + } + } + + if (context.Principal.Identity is not ClaimsIdentity identity) + { + return; + } + + var providerName = context.Principal.GetClaim(StellaOpsClaimTypes.IdentityProvider); + if (string.IsNullOrWhiteSpace(providerName)) + { + return; + } + + if (!registry.TryGet(providerName, out var provider)) + { + context.Reject(OpenIddictConstants.Errors.InvalidToken, "The identity provider associated with the token is unavailable."); + logger.LogWarning("Access token validation failed: provider {Provider} unavailable for subject {Subject}.", providerName, context.Principal.GetClaim(OpenIddictConstants.Claims.Subject)); + return; + } + + AuthorityUserDescriptor? user = null; + AuthorityClientDescriptor? client = null; + + var subject = context.Principal.GetClaim(OpenIddictConstants.Claims.Subject); + if (!string.IsNullOrWhiteSpace(subject)) + { + user = await provider.Credentials.FindBySubjectAsync(subject, context.CancellationToken).ConfigureAwait(false); + if (user is null) + { + context.Reject(OpenIddictConstants.Errors.InvalidToken, "The subject referenced by the token no longer exists."); + logger.LogWarning("Access token validation failed: subject {SubjectId} not found.", subject); + return; + } + activity?.SetTag("authority.subject_id", subject); + } + + if (!string.IsNullOrWhiteSpace(clientId) && provider.ClientProvisioning is not null) + { + client = await provider.ClientProvisioning.FindByClientIdAsync(clientId, context.CancellationToken).ConfigureAwait(false); + } + + var enrichmentContext = new AuthorityClaimsEnrichmentContext(provider.Context, user, client); + await provider.ClaimsEnricher.EnrichAsync(identity, enrichmentContext, context.CancellationToken).ConfigureAwait(false); + logger.LogInformation("Access token validated for subject {Subject} and client {ClientId}.", + identity.GetClaim(OpenIddictConstants.Claims.Subject), + identity.GetClaim(OpenIddictConstants.Claims.ClientId)); + } +} diff --git a/src/StellaOps.Authority/StellaOps.Authority/Plugins/AuthorityPluginLoader.cs b/src/StellaOps.Authority/StellaOps.Authority/Plugins/AuthorityPluginLoader.cs new file mode 100644 index 00000000..0dfd578a --- /dev/null +++ b/src/StellaOps.Authority/StellaOps.Authority/Plugins/AuthorityPluginLoader.cs @@ -0,0 +1,225 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Reflection; +using Microsoft.Extensions.Configuration; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Logging; +using StellaOps.Authority.Plugins.Abstractions; +using StellaOps.Plugin.Hosting; + +namespace StellaOps.Authority.Plugins; + +internal static class AuthorityPluginLoader +{ + public static AuthorityPluginRegistrationSummary RegisterPlugins( + IServiceCollection services, + IConfiguration configuration, + PluginHostOptions hostOptions, + IReadOnlyCollection pluginContexts, + ILogger? logger) + { + ArgumentNullException.ThrowIfNull(services); + ArgumentNullException.ThrowIfNull(configuration); + ArgumentNullException.ThrowIfNull(hostOptions); + ArgumentNullException.ThrowIfNull(pluginContexts); + + if (pluginContexts.Count == 0) + { + return AuthorityPluginRegistrationSummary.Empty; + } + + var loadResult = PluginHost.LoadPlugins(hostOptions, logger); + var descriptors = loadResult.Plugins + .Select(p => new LoadedPluginDescriptor(p.Assembly, p.AssemblyPath)) + .ToArray(); + + return RegisterPluginsCore( + services, + configuration, + pluginContexts, + descriptors, + loadResult.MissingOrderedPlugins, + logger); + } + + internal static AuthorityPluginRegistrationSummary RegisterPluginsCore( + IServiceCollection services, + IConfiguration configuration, + IReadOnlyCollection pluginContexts, + IReadOnlyCollection loadedAssemblies, + IReadOnlyCollection missingOrdered, + ILogger? logger) + { + var registrarLookup = DiscoverRegistrars(loadedAssemblies, logger); + var registered = new List(); + var failures = new List(); + + foreach (var pluginContext in pluginContexts) + { + var manifest = pluginContext.Manifest; + + if (!manifest.Enabled) + { + logger?.LogInformation( + "Skipping disabled Authority plugin '{PluginName}' ({PluginType}).", + manifest.Name, + manifest.Type); + continue; + } + + if (!IsAssemblyLoaded(manifest, loadedAssemblies)) + { + var reason = $"Assembly '{manifest.AssemblyName ?? manifest.AssemblyPath ?? manifest.Type}' was not loaded."; + logger?.LogError( + "Failed to register Authority plugin '{PluginName}': {Reason}", + manifest.Name, + reason); + failures.Add(new AuthorityPluginRegistrationFailure(manifest.Name, reason)); + continue; + } + + if (!registrarLookup.TryGetValue(manifest.Type, out var registrar)) + { + var reason = $"No registrar found for plugin type '{manifest.Type}'."; + logger?.LogError( + "Failed to register Authority plugin '{PluginName}': {Reason}", + manifest.Name, + reason); + failures.Add(new AuthorityPluginRegistrationFailure(manifest.Name, reason)); + continue; + } + + try + { + registrar.Register(new AuthorityPluginRegistrationContext(services, pluginContext, configuration)); + registered.Add(manifest.Name); + + logger?.LogInformation( + "Registered Authority plugin '{PluginName}' ({PluginType}).", + manifest.Name, + manifest.Type); + } + catch (Exception ex) + { + var reason = $"Registration threw {ex.GetType().Name}."; + logger?.LogError( + ex, + "Failed to register Authority plugin '{PluginName}'.", + manifest.Name); + failures.Add(new AuthorityPluginRegistrationFailure(manifest.Name, reason)); + } + } + + if (missingOrdered.Count > 0) + { + foreach (var missing in missingOrdered) + { + logger?.LogWarning( + "Configured plugin '{PluginName}' was not found in the plugin directory.", + missing); + } + } + + return new AuthorityPluginRegistrationSummary(registered, failures, missingOrdered); + } + + private static Dictionary DiscoverRegistrars( + IReadOnlyCollection loadedAssemblies, + ILogger? logger) + { + var lookup = new Dictionary(StringComparer.OrdinalIgnoreCase); + + foreach (var descriptor in loadedAssemblies) + { + foreach (var type in GetLoadableTypes(descriptor.Assembly)) + { + if (!typeof(IAuthorityPluginRegistrar).IsAssignableFrom(type) || type.IsAbstract || type.IsInterface) + { + continue; + } + + try + { + if (Activator.CreateInstance(type) is not IAuthorityPluginRegistrar registrar) + { + continue; + } + + if (string.IsNullOrWhiteSpace(registrar.PluginType)) + { + logger?.LogWarning( + "Authority plugin registrar '{RegistrarType}' returned an empty plugin type and will be ignored.", + type.FullName); + continue; + } + + if (lookup.TryGetValue(registrar.PluginType, out var existing)) + { + logger?.LogWarning( + "Multiple registrars detected for plugin type '{PluginType}'. Replacing '{ExistingType}' with '{RegistrarType}'.", + registrar.PluginType, + existing.GetType().FullName, + type.FullName); + } + + lookup[registrar.PluginType] = registrar; + } + catch (Exception ex) + { + logger?.LogError( + ex, + "Failed to instantiate Authority plugin registrar '{RegistrarType}'.", + type.FullName); + } + } + } + + return lookup; + } + + private static bool IsAssemblyLoaded( + AuthorityPluginManifest manifest, + IReadOnlyCollection loadedAssemblies) + { + if (!string.IsNullOrWhiteSpace(manifest.AssemblyName) && + loadedAssemblies.Any(descriptor => + string.Equals( + descriptor.Assembly.GetName().Name, + manifest.AssemblyName, + StringComparison.OrdinalIgnoreCase))) + { + return true; + } + + if (!string.IsNullOrWhiteSpace(manifest.AssemblyPath) && + loadedAssemblies.Any(descriptor => + string.Equals( + descriptor.AssemblyPath, + manifest.AssemblyPath, + StringComparison.OrdinalIgnoreCase))) + { + return true; + } + + // As a fallback, assume any loaded assembly whose simple name contains the plugin type is a match. + return loadedAssemblies.Any(descriptor => + descriptor.Assembly.GetName().Name?.Contains(manifest.Type, StringComparison.OrdinalIgnoreCase) == true); + } + + private static IEnumerable GetLoadableTypes(Assembly assembly) + { + try + { + return assembly.GetTypes(); + } + catch (ReflectionTypeLoadException ex) + { + return ex.Types.Where(static type => type is not null)!; + } + } + + internal readonly record struct LoadedPluginDescriptor( + Assembly Assembly, + string AssemblyPath); +} diff --git a/src/StellaOps.Authority/StellaOps.Authority/Plugins/AuthorityPluginRegistrationSummary.cs b/src/StellaOps.Authority/StellaOps.Authority/Plugins/AuthorityPluginRegistrationSummary.cs new file mode 100644 index 00000000..08e01501 --- /dev/null +++ b/src/StellaOps.Authority/StellaOps.Authority/Plugins/AuthorityPluginRegistrationSummary.cs @@ -0,0 +1,20 @@ +using System; +using System.Collections.Generic; + +namespace StellaOps.Authority.Plugins; + +internal sealed record AuthorityPluginRegistrationSummary( + IReadOnlyCollection RegisteredPlugins, + IReadOnlyCollection Failures, + IReadOnlyCollection MissingOrderedPlugins) +{ + public static AuthorityPluginRegistrationSummary Empty { get; } = + new AuthorityPluginRegistrationSummary( + Array.Empty(), + Array.Empty(), + Array.Empty()); + + public bool HasFailures => Failures.Count > 0 || MissingOrderedPlugins.Count > 0; +} + +internal sealed record AuthorityPluginRegistrationFailure(string PluginName, string Reason); diff --git a/src/StellaOps.Authority/StellaOps.Authority/Program.cs b/src/StellaOps.Authority/StellaOps.Authority/Program.cs new file mode 100644 index 00000000..9a35e0a8 --- /dev/null +++ b/src/StellaOps.Authority/StellaOps.Authority/Program.cs @@ -0,0 +1,491 @@ +using System.Diagnostics; +using Microsoft.AspNetCore.Builder; +using Microsoft.AspNetCore.Http; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Configuration; +using Microsoft.Extensions.Hosting; +using Microsoft.Extensions.Options; +using Microsoft.AspNetCore.RateLimiting; +using Microsoft.Extensions.Logging.Abstractions; +using OpenIddict.Abstractions; +using OpenIddict.Server; +using OpenIddict.Server.AspNetCore; +using MongoDB.Driver; +using Serilog; +using Serilog.Events; +using StellaOps.Authority; +using StellaOps.Authority.Plugins.Abstractions; +using StellaOps.Authority.Plugins; +using StellaOps.Authority.Bootstrap; +using StellaOps.Authority.Storage.Mongo.Extensions; +using StellaOps.Authority.Storage.Mongo.Initialization; +using StellaOps.Configuration; +using StellaOps.Plugin.DependencyInjection; +using StellaOps.Plugin.Hosting; +using StellaOps.Authority.OpenIddict.Handlers; +using System.Linq; + +var builder = WebApplication.CreateBuilder(args); + +Activity.DefaultIdFormat = ActivityIdFormat.W3C; +Activity.ForceDefaultIdFormat = true; + +AuthorityTelemetryConfiguration.Configure(builder); + +var authorityConfiguration = StellaOpsAuthorityConfiguration.Build(options => +{ + options.BasePath = builder.Environment.ContentRootPath; + options.EnvironmentPrefix = "STELLAOPS_AUTHORITY_"; + options.ConfigureBuilder = configurationBuilder => + { + var contentRoot = builder.Environment.ContentRootPath; + foreach (var relative in new[] + { + "../etc/authority.yaml", + "../etc/authority.local.yaml", + "authority.yaml", + "authority.local.yaml" + }) + { + var path = Path.Combine(contentRoot, relative); + configurationBuilder.AddYamlFile(path, optional: true); + } + }; +}); + +builder.Configuration.AddConfiguration(authorityConfiguration.Configuration); + +builder.Host.UseSerilog((context, _, loggerConfiguration) => +{ + loggerConfiguration + .ReadFrom.Configuration(context.Configuration) + .Enrich.FromLogContext() + .MinimumLevel.Override("Microsoft.AspNetCore.Hosting.Diagnostics", LogEventLevel.Warning) + .WriteTo.Console(); +}); + +var authorityOptions = authorityConfiguration.Options; +var issuer = authorityOptions.Issuer ?? throw new InvalidOperationException("Authority issuer configuration is required."); +builder.Services.AddSingleton(authorityOptions); +builder.Services.AddSingleton>(Options.Create(authorityOptions)); + +builder.Services.AddRateLimiter(rateLimiterOptions => +{ + AuthorityRateLimiter.Configure(rateLimiterOptions, authorityOptions); +}); + +AuthorityPluginContext[] pluginContexts = AuthorityPluginConfigurationLoader + .Load(authorityOptions, builder.Environment.ContentRootPath) + .ToArray(); + +builder.Services.AddSingleton>(pluginContexts); +builder.Services.AddSingleton(_ => new AuthorityPluginRegistry(pluginContexts)); + +var pluginHostOptions = BuildPluginHostOptions(authorityOptions, builder.Environment.ContentRootPath); +builder.Services.AddSingleton(pluginHostOptions); +builder.Services.RegisterPluginRoutines(authorityConfiguration.Configuration, pluginHostOptions); + +builder.Services.AddAuthorityMongoStorage(storageOptions => +{ + storageOptions.ConnectionString = authorityOptions.Storage.ConnectionString; + storageOptions.DatabaseName = authorityOptions.Storage.DatabaseName; + storageOptions.CommandTimeout = authorityOptions.Storage.CommandTimeout; +}); + +builder.Services.AddSingleton(); +builder.Services.AddScoped(); +builder.Services.AddScoped(); +builder.Services.AddScoped(); +builder.Services.AddScoped(); +builder.Services.AddScoped(); + +var pluginRegistrationSummary = AuthorityPluginLoader.RegisterPlugins( + builder.Services, + authorityConfiguration.Configuration, + pluginHostOptions, + pluginContexts, + NullLogger.Instance); + +builder.Services.AddSingleton(pluginRegistrationSummary); + +builder.Services.AddRouting(options => options.LowercaseUrls = true); +builder.Services.AddProblemDetails(); +builder.Services.AddAuthentication(); +builder.Services.AddAuthorization(); + +builder.Services.AddOpenIddict() + .AddServer(options => + { + options.SetIssuer(issuer); + options.SetTokenEndpointUris("/token"); + options.SetAuthorizationEndpointUris("/authorize"); + options.SetIntrospectionEndpointUris("/introspect"); + options.SetRevocationEndpointUris("/revoke"); + options.SetJsonWebKeySetEndpointUris("/jwks"); + + options.AllowPasswordFlow(); + options.AllowClientCredentialsFlow(); + options.AllowRefreshTokenFlow(); + + options.SetAccessTokenLifetime(authorityOptions.AccessTokenLifetime); + options.SetRefreshTokenLifetime(authorityOptions.RefreshTokenLifetime); + options.SetIdentityTokenLifetime(authorityOptions.IdentityTokenLifetime); + options.SetAuthorizationCodeLifetime(authorityOptions.AuthorizationCodeLifetime); + options.SetDeviceCodeLifetime(authorityOptions.DeviceCodeLifetime); + + options.DisableAccessTokenEncryption(); + options.DisableTokenStorage(); + options.DisableAuthorizationStorage(); + + options.RegisterScopes( + OpenIddictConstants.Scopes.OpenId, + OpenIddictConstants.Scopes.Email, + OpenIddictConstants.Scopes.Profile, + OpenIddictConstants.Scopes.OfflineAccess); + + options.AddEphemeralEncryptionKey() + .AddEphemeralSigningKey(); + + var aspNetCoreBuilder = options.UseAspNetCore() + .EnableAuthorizationEndpointPassthrough() + .EnableTokenEndpointPassthrough(); + + if (builder.Environment.IsDevelopment()) + { + aspNetCoreBuilder.DisableTransportSecurityRequirement(); + } + + options.AddEventHandler(descriptor => + { + descriptor.UseScopedHandler(); + }); + + options.AddEventHandler(descriptor => + { + descriptor.UseScopedHandler(); + }); + + options.AddEventHandler(descriptor => + { + descriptor.UseScopedHandler(); + }); + + options.AddEventHandler(descriptor => + { + descriptor.UseScopedHandler(); + }); + + options.AddEventHandler(descriptor => + { + descriptor.UseScopedHandler(); + }); + }); + +builder.Services.Configure(options => +{ + options.DisableSlidingRefreshTokenExpiration = false; + options.DisableRollingRefreshTokens = false; +}); + +var app = builder.Build(); + +var mongoInitializer = app.Services.GetRequiredService(); +var mongoDatabase = app.Services.GetRequiredService(); +await mongoInitializer.InitialiseAsync(mongoDatabase, CancellationToken.None); + +var registrationSummary = app.Services.GetRequiredService(); +if (registrationSummary.RegisteredPlugins.Count > 0) +{ + app.Logger.LogInformation( + "Authority plugins registered: {Plugins}", + string.Join(", ", registrationSummary.RegisteredPlugins)); +} + +foreach (var failure in registrationSummary.Failures) +{ + app.Logger.LogError( + "Authority plugin '{PluginName}' failed to register: {Reason}", + failure.PluginName, + failure.Reason); +} + +foreach (var missing in registrationSummary.MissingOrderedPlugins) +{ + app.Logger.LogWarning( + "Configured Authority plugin '{PluginName}' was not discovered during startup.", + missing); +} + +var identityProviderRegistry = app.Services.GetRequiredService(); +if (identityProviderRegistry.Providers.Count == 0) +{ + app.Logger.LogWarning("No identity provider plugins were registered."); +} +else +{ + foreach (var provider in identityProviderRegistry.Providers) + { + var caps = provider.Capabilities; + app.Logger.LogInformation( + "Identity provider plugin '{PluginName}' (type {PluginType}) capabilities: password={Password}, mfa={Mfa}, clientProvisioning={ClientProvisioning}.", + provider.Name, + provider.Type, + caps.SupportsPassword, + caps.SupportsMfa, + caps.SupportsClientProvisioning); + } +} + +if (authorityOptions.Bootstrap.Enabled) +{ + var bootstrapGroup = app.MapGroup("/internal"); + bootstrapGroup.AddEndpointFilter(new BootstrapApiKeyFilter(authorityOptions)); + + bootstrapGroup.MapPost("/users", async ( + BootstrapUserRequest request, + IAuthorityIdentityProviderRegistry registry, + CancellationToken cancellationToken) => + { + if (request is null) + { + return Results.BadRequest(new { error = "invalid_request", message = "Request payload is required." }); + } + + var providerName = string.IsNullOrWhiteSpace(request.Provider) + ? authorityOptions.Bootstrap.DefaultIdentityProvider + : request.Provider; + + if (string.IsNullOrWhiteSpace(providerName) || !registry.TryGet(providerName!, out var provider)) + { + return Results.BadRequest(new { error = "invalid_provider", message = "Specified identity provider was not found." }); + } + + if (!provider.Capabilities.SupportsPassword) + { + return Results.BadRequest(new { error = "unsupported_provider", message = "Selected provider does not support password provisioning." }); + } + + if (string.IsNullOrWhiteSpace(request.Username) || string.IsNullOrEmpty(request.Password)) + { + return Results.BadRequest(new { error = "invalid_request", message = "Username and password are required." }); + } + + var roles = request.Roles is null ? Array.Empty() : request.Roles.ToArray(); + var attributes = request.Attributes is null + ? new Dictionary(StringComparer.OrdinalIgnoreCase) + : new Dictionary(request.Attributes, StringComparer.OrdinalIgnoreCase); + + var registration = new AuthorityUserRegistration( + request.Username, + request.Password, + request.DisplayName, + request.Email, + request.RequirePasswordReset, + roles, + attributes); + + var result = await provider.Credentials.UpsertUserAsync(registration, cancellationToken).ConfigureAwait(false); + + if (!result.Succeeded || result.Value is null) + { + return Results.BadRequest(new { error = result.ErrorCode ?? "bootstrap_failed", message = result.Message ?? "User provisioning failed." }); + } + + return Results.Ok(new + { + provider = provider.Name, + subjectId = result.Value.SubjectId, + username = result.Value.Username + }); + }); + + bootstrapGroup.MapPost("/clients", async ( + BootstrapClientRequest request, + IAuthorityIdentityProviderRegistry registry, + CancellationToken cancellationToken) => + { + if (request is null) + { + return Results.BadRequest(new { error = "invalid_request", message = "Request payload is required." }); + } + + var providerName = string.IsNullOrWhiteSpace(request.Provider) + ? authorityOptions.Bootstrap.DefaultIdentityProvider + : request.Provider; + + if (string.IsNullOrWhiteSpace(providerName) || !registry.TryGet(providerName!, out var provider)) + { + return Results.BadRequest(new { error = "invalid_provider", message = "Specified identity provider was not found." }); + } + + if (!provider.Capabilities.SupportsClientProvisioning || provider.ClientProvisioning is null) + { + return Results.BadRequest(new { error = "unsupported_provider", message = "Selected provider does not support client provisioning." }); + } + + if (string.IsNullOrWhiteSpace(request.ClientId)) + { + return Results.BadRequest(new { error = "invalid_request", message = "ClientId is required." }); + } + + if (request.Confidential && string.IsNullOrWhiteSpace(request.ClientSecret)) + { + return Results.BadRequest(new { error = "invalid_request", message = "Confidential clients require a client secret." }); + } + + if (!TryParseUris(request.RedirectUris, out var redirectUris, out var redirectError)) + { + return Results.BadRequest(new { error = "invalid_request", message = redirectError }); + } + + if (!TryParseUris(request.PostLogoutRedirectUris, out var postLogoutUris, out var postLogoutError)) + { + return Results.BadRequest(new { error = "invalid_request", message = postLogoutError }); + } + + var properties = request.Properties is null + ? new Dictionary(StringComparer.OrdinalIgnoreCase) + : new Dictionary(request.Properties, StringComparer.OrdinalIgnoreCase); + + var registration = new AuthorityClientRegistration( + request.ClientId, + request.Confidential, + request.DisplayName, + request.ClientSecret, + request.AllowedGrantTypes ?? Array.Empty(), + request.AllowedScopes ?? Array.Empty(), + redirectUris, + postLogoutUris, + properties); + + var result = await provider.ClientProvisioning.CreateOrUpdateAsync(registration, cancellationToken).ConfigureAwait(false); + + if (!result.Succeeded || result.Value is null) + { + return Results.BadRequest(new { error = result.ErrorCode ?? "bootstrap_failed", message = result.Message ?? "Client provisioning failed." }); + } + + return Results.Ok(new + { + provider = provider.Name, + clientId = result.Value.ClientId, + confidential = result.Value.Confidential + }); + }); +} + +app.UseSerilogRequestLogging(options => +{ + options.EnrichDiagnosticContext = (diagnosticContext, httpContext) => + { + diagnosticContext.Set("TraceId", Activity.Current?.TraceId.ToString()); + diagnosticContext.Set("UserAgent", httpContext.Request.Headers.UserAgent.ToString()); + }; +}); + +app.UseExceptionHandler(static errorApp => +{ + errorApp.Run(async context => + { + context.Response.ContentType = "application/problem+json"; + var problem = Results.Problem( + statusCode: StatusCodes.Status500InternalServerError, + title: "Unhandled server error", + detail: "Unexpected failure while processing the request."); + + await problem.ExecuteAsync(context); + }); +}); + +app.UseRouting(); +app.UseRateLimiter(); +app.UseAuthentication(); +app.UseAuthorization(); + +app.MapGet("/health", async (IAuthorityIdentityProviderRegistry registry, CancellationToken cancellationToken) => + { + var pluginHealth = new List(); + foreach (var provider in registry.Providers) + { + var health = await provider.CheckHealthAsync(cancellationToken).ConfigureAwait(false); + pluginHealth.Add(new + { + provider = provider.Name, + status = health.Status.ToString().ToLowerInvariant(), + message = health.Message + }); + } + + return Results.Ok(new + { + status = "healthy", + identityProviders = pluginHealth + }); + }) + .WithName("HealthCheck"); + +app.MapGet("/ready", (IAuthorityIdentityProviderRegistry registry) => + Results.Ok(new + { + status = registry.Providers.Count > 0 ? "ready" : "degraded", + identityProviders = registry.Providers.Select(p => p.Name).ToArray() + })) + .WithName("ReadinessCheck"); + +app.Run(); + +static PluginHostOptions BuildPluginHostOptions(StellaOpsAuthorityOptions options, string basePath) +{ + var pluginDirectory = options.PluginDirectories.FirstOrDefault(); + var hostOptions = new PluginHostOptions + { + BaseDirectory = basePath, + PluginsDirectory = string.IsNullOrWhiteSpace(pluginDirectory) + ? Path.Combine("PluginBinaries", "Authority") + : pluginDirectory, + PrimaryPrefix = "StellaOps.Authority" + }; + + if (!hostOptions.SearchPatterns.Any(pattern => string.Equals(pattern, "StellaOps.Authority.Plugin.*.dll", StringComparison.OrdinalIgnoreCase))) + { + hostOptions.SearchPatterns.Add("StellaOps.Authority.Plugin.*.dll"); + } + + foreach (var pair in options.Plugins.Descriptors.OrderBy(static p => p.Key, StringComparer.OrdinalIgnoreCase)) + { + var descriptor = pair.Value; + if (descriptor.Enabled && !string.IsNullOrWhiteSpace(descriptor.AssemblyName)) + { + hostOptions.PluginOrder.Add(descriptor.AssemblyName!); + } + } + + return hostOptions; +} + +static bool TryParseUris(IReadOnlyCollection? values, out IReadOnlyCollection uris, out string? error) +{ + error = null; + + if (values is null || values.Count == 0) + { + uris = Array.Empty(); + return true; + } + + var parsed = new List(values.Count); + foreach (var entry in values) + { + if (string.IsNullOrWhiteSpace(entry) || !Uri.TryCreate(entry, UriKind.Absolute, out var uri)) + { + uris = Array.Empty(); + error = $"Invalid URI value '{entry}'."; + return false; + } + + parsed.Add(uri); + } + + uris = parsed; + return true; +} diff --git a/src/StellaOps.Authority/StellaOps.Authority/Properties/AssemblyInfo.cs b/src/StellaOps.Authority/StellaOps.Authority/Properties/AssemblyInfo.cs new file mode 100644 index 00000000..a7d9fe65 --- /dev/null +++ b/src/StellaOps.Authority/StellaOps.Authority/Properties/AssemblyInfo.cs @@ -0,0 +1,3 @@ +using System.Runtime.CompilerServices; + +[assembly: InternalsVisibleTo("StellaOps.Authority.Tests")] diff --git a/src/StellaOps.Authority/StellaOps.Authority/Properties/launchSettings.json b/src/StellaOps.Authority/StellaOps.Authority/Properties/launchSettings.json new file mode 100644 index 00000000..238d8956 --- /dev/null +++ b/src/StellaOps.Authority/StellaOps.Authority/Properties/launchSettings.json @@ -0,0 +1,23 @@ +{ + "$schema": "https://json.schemastore.org/launchsettings.json", + "profiles": { + "http": { + "commandName": "Project", + "dotnetRunMessages": true, + "launchBrowser": true, + "applicationUrl": "http://localhost:5165", + "environmentVariables": { + "ASPNETCORE_ENVIRONMENT": "Development" + } + }, + "https": { + "commandName": "Project", + "dotnetRunMessages": true, + "launchBrowser": true, + "applicationUrl": "https://localhost:7182;http://localhost:5165", + "environmentVariables": { + "ASPNETCORE_ENVIRONMENT": "Development" + } + } + } +} diff --git a/src/StellaOps.Authority/StellaOps.Authority/RateLimiting/AuthorityRateLimiterFeature.cs b/src/StellaOps.Authority/StellaOps.Authority/RateLimiting/AuthorityRateLimiterFeature.cs new file mode 100644 index 00000000..35fdf25c --- /dev/null +++ b/src/StellaOps.Authority/StellaOps.Authority/RateLimiting/AuthorityRateLimiterFeature.cs @@ -0,0 +1,26 @@ +using System; + +namespace StellaOps.Authority.RateLimiting; + +/// +/// Feature carrying rate limiter metadata for the current HTTP request. +/// +internal sealed class AuthorityRateLimiterFeature : IAuthorityRateLimiterFeature +{ + public AuthorityRateLimiterFeature() + { + Metadata = new AuthorityRateLimiterMetadata(); + } + + public AuthorityRateLimiterFeature(AuthorityRateLimiterMetadata metadata) + { + Metadata = metadata ?? throw new ArgumentNullException(nameof(metadata)); + } + + public AuthorityRateLimiterMetadata Metadata { get; } +} + +internal interface IAuthorityRateLimiterFeature +{ + AuthorityRateLimiterMetadata Metadata { get; } +} diff --git a/src/StellaOps.Authority/StellaOps.Authority/RateLimiting/AuthorityRateLimiterMetadata.cs b/src/StellaOps.Authority/StellaOps.Authority/RateLimiting/AuthorityRateLimiterMetadata.cs new file mode 100644 index 00000000..62dee1ce --- /dev/null +++ b/src/StellaOps.Authority/StellaOps.Authority/RateLimiting/AuthorityRateLimiterMetadata.cs @@ -0,0 +1,64 @@ +using System.Collections.Generic; + +namespace StellaOps.Authority.RateLimiting; + +/// +/// Metadata captured for the current request to assist rate limiter partitioning and diagnostics. +/// +internal sealed class AuthorityRateLimiterMetadata +{ + private static readonly StringComparer OrdinalIgnoreCase = StringComparer.OrdinalIgnoreCase; + + private readonly Dictionary tags = new(OrdinalIgnoreCase); + + /// + /// Canonical endpoint associated with the request (e.g. "/token"). + /// + public string? Endpoint { get; set; } + + /// + /// Remote IP address observed for the request (post proxy resolution where available). + /// + public string? RemoteIp { get; set; } + + /// + /// Forwarded IP address extracted from proxy headers (if present). + /// + public string? ForwardedFor { get; set; } + + /// + /// OAuth client identifier associated with the request, when available. + /// + public string? ClientId { get; set; } + + /// + /// Subject identifier (user) associated with the request, when available. + /// + public string? SubjectId { get; set; } + + /// + /// Additional metadata tags that can be attached by later handlers. + /// + public IReadOnlyDictionary Tags => tags; + + /// + /// Adds or updates an arbitrary metadata tag for downstream consumers. + /// + /// The tag name. + /// The tag value (removed when null/whitespace). + public void SetTag(string name, string? value) + { + if (string.IsNullOrWhiteSpace(name)) + { + return; + } + + if (string.IsNullOrWhiteSpace(value)) + { + tags.Remove(name); + return; + } + + tags[name] = value; + } +} diff --git a/src/StellaOps.Authority/StellaOps.Authority/RateLimiting/AuthorityRateLimiterMetadataAccessor.cs b/src/StellaOps.Authority/StellaOps.Authority/RateLimiting/AuthorityRateLimiterMetadataAccessor.cs new file mode 100644 index 00000000..ebd5e012 --- /dev/null +++ b/src/StellaOps.Authority/StellaOps.Authority/RateLimiting/AuthorityRateLimiterMetadataAccessor.cs @@ -0,0 +1,83 @@ +using System; +using Microsoft.AspNetCore.Http; + +namespace StellaOps.Authority.RateLimiting; + +/// +/// Provides access to the rate limiter metadata for the current HTTP request. +/// +internal interface IAuthorityRateLimiterMetadataAccessor +{ + /// + /// Retrieves the metadata for the current request, if available. + /// + /// The metadata instance or null when no HTTP context is present. + AuthorityRateLimiterMetadata? GetMetadata(); + + /// + /// Updates the client identifier associated with the current request. + /// + void SetClientId(string? clientId); + + /// + /// Updates the subject identifier associated with the current request. + /// + void SetSubjectId(string? subjectId); + + /// + /// Adds or removes a metadata tag for the current request. + /// + void SetTag(string name, string? value); +} + +internal sealed class AuthorityRateLimiterMetadataAccessor : IAuthorityRateLimiterMetadataAccessor +{ + private readonly IHttpContextAccessor httpContextAccessor; + + public AuthorityRateLimiterMetadataAccessor(IHttpContextAccessor httpContextAccessor) + { + this.httpContextAccessor = httpContextAccessor ?? throw new ArgumentNullException(nameof(httpContextAccessor)); + } + + public AuthorityRateLimiterMetadata? GetMetadata() + { + return TryGetMetadata(); + } + + public void SetClientId(string? clientId) + { + var metadata = TryGetMetadata(); + if (metadata is not null) + { + metadata.ClientId = Normalize(clientId); + metadata.SetTag("authority.client_id", metadata.ClientId); + } + } + + public void SetSubjectId(string? subjectId) + { + var metadata = TryGetMetadata(); + if (metadata is not null) + { + metadata.SubjectId = Normalize(subjectId); + metadata.SetTag("authority.subject_id", metadata.SubjectId); + } + } + + public void SetTag(string name, string? value) + { + var metadata = TryGetMetadata(); + metadata?.SetTag(name, value); + } + + private AuthorityRateLimiterMetadata? TryGetMetadata() + { + var context = httpContextAccessor.HttpContext; + return context?.Features.Get()?.Metadata; + } + + private static string? Normalize(string? value) + { + return string.IsNullOrWhiteSpace(value) ? null : value; + } +} diff --git a/src/StellaOps.Authority/StellaOps.Authority/RateLimiting/AuthorityRateLimiterMetadataMiddleware.cs b/src/StellaOps.Authority/StellaOps.Authority/RateLimiting/AuthorityRateLimiterMetadataMiddleware.cs new file mode 100644 index 00000000..921c147e --- /dev/null +++ b/src/StellaOps.Authority/StellaOps.Authority/RateLimiting/AuthorityRateLimiterMetadataMiddleware.cs @@ -0,0 +1,278 @@ +using System; +using System.Globalization; +using System.IO; +using System.Net; +using System.Text; +using System.Threading.Tasks; +using Microsoft.AspNetCore.Http; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Primitives; + +namespace StellaOps.Authority.RateLimiting; + +/// +/// Captures request metadata required for rate limiting decisions before the rate limiter middleware executes. +/// +internal sealed class AuthorityRateLimiterMetadataMiddleware +{ + private static readonly string[] ForwardedHeaders = + { + "X-Forwarded-For", + "X-Real-IP", + "Forwarded" + }; + + private readonly RequestDelegate next; + private readonly TimeProvider clock; + private readonly ILogger logger; + + public AuthorityRateLimiterMetadataMiddleware( + RequestDelegate next, + TimeProvider clock, + ILogger logger) + { + this.next = next ?? throw new ArgumentNullException(nameof(next)); + this.clock = clock ?? throw new ArgumentNullException(nameof(clock)); + this.logger = logger ?? throw new ArgumentNullException(nameof(logger)); + } + + public async Task InvokeAsync(HttpContext context) + { + ArgumentNullException.ThrowIfNull(context); + + var remoteAddress = NormalizeAddress(context.Connection.RemoteIpAddress); + var forwardedFor = ResolveForwardedIp(context.Request.Headers); + + var metadata = new AuthorityRateLimiterMetadata + { + Endpoint = NormalizeEndpoint(context.Request.Path), + RemoteIp = remoteAddress ?? forwardedFor, + ForwardedFor = forwardedFor, + }; + + context.Features.Set(new AuthorityRateLimiterFeature(metadata)); + + if (string.Equals(metadata.Endpoint, "/token", StringComparison.Ordinal)) + { + metadata.ClientId = await ResolveTokenClientIdAsync(context).ConfigureAwait(false); + } + else if (string.Equals(metadata.Endpoint, "/authorize", StringComparison.Ordinal)) + { + metadata.ClientId = ResolveAuthorizeClientId(context.Request.Query); + } + + if (!string.IsNullOrWhiteSpace(metadata.ClientId)) + { + metadata.SetTag("authority.client_id", metadata.ClientId); + } + + if (!string.IsNullOrWhiteSpace(metadata.ForwardedFor)) + { + metadata.SetTag("authority.forwarded_for", metadata.ForwardedFor!); + } + + metadata.SetTag("authority.endpoint", metadata.Endpoint ?? string.Empty); + metadata.SetTag("authority.remote_ip", metadata.RemoteIp ?? "unknown"); + metadata.SetTag("authority.captured_at", clock.GetUtcNow().ToString("O", CultureInfo.InvariantCulture)); + + await next(context).ConfigureAwait(false); + } + + private static string? NormalizeEndpoint(PathString path) + { + if (!path.HasValue || string.IsNullOrWhiteSpace(path.Value)) + { + return "/"; + } + + return path.Value!.TrimEnd('/').ToLowerInvariant(); + } + + private static string? NormalizeAddress(IPAddress? address) + { + if (address is null) + { + return null; + } + + if (IPAddress.Any.Equals(address) || IPAddress.IPv6Any.Equals(address) || + IPAddress.IPv6None.Equals(address) || IPAddress.None.Equals(address)) + { + return null; + } + + return address.ToString(); + } + + private static string? ResolveForwardedIp(IHeaderDictionary headers) + { + foreach (var headerName in ForwardedHeaders) + { + if (!headers.TryGetValue(headerName, out var values) || StringValues.IsNullOrEmpty(values)) + { + continue; + } + + var raw = values.ToString(); + if (string.Equals(headerName, "Forwarded", StringComparison.OrdinalIgnoreCase)) + { + var forwarded = ForwardedHeaderParser.Parse(raw); + if (!string.IsNullOrWhiteSpace(forwarded)) + { + return forwarded; + } + } + else + { + var candidate = raw.Split(',')[0].Trim(); + if (!string.IsNullOrWhiteSpace(candidate)) + { + return candidate; + } + } + } + + return null; + } + + private static string? ResolveAuthorizeClientId(IQueryCollection query) + { + if (query.TryGetValue("client_id", out var values) && !StringValues.IsNullOrEmpty(values)) + { + return values.ToString(); + } + + return null; + } + + private async Task ResolveTokenClientIdAsync(HttpContext context) + { + var request = context.Request; + + if (TryParseBasicClientId(request.Headers["Authorization"], out var clientId)) + { + return clientId; + } + + if (!HttpMethods.IsPost(request.Method) || !request.HasFormContentType) + { + return null; + } + + try + { + request.EnableBuffering(); + var form = await request.ReadFormAsync(context.RequestAborted).ConfigureAwait(false); + if (form.TryGetValue("client_id", out var values) && !StringValues.IsNullOrEmpty(values)) + { + return values.ToString(); + } + } + catch (InvalidDataException ex) + { + logger.LogDebug(ex, "Failed to parse form data while resolving token endpoint client_id."); + } + finally + { + if (request.Body.CanSeek) + { + request.Body.Seek(0, SeekOrigin.Begin); + } + } + + return null; + } + + private static bool TryParseBasicClientId(StringValues authorizationHeader, out string? clientId) + { + clientId = null; + + if (StringValues.IsNullOrEmpty(authorizationHeader)) + { + return false; + } + + for (var index = 0; index < authorizationHeader.Count; index++) + { + var value = authorizationHeader[index]; + if (string.IsNullOrWhiteSpace(value) || !value.StartsWith("Basic ", StringComparison.OrdinalIgnoreCase)) + { + continue; + } + + var span = value.AsSpan(6).Trim(); + if (span.IsEmpty) + { + continue; + } + + if (TryDecodeBase64(span, out var decoded)) + { + var separator = decoded.IndexOf(':'); + clientId = separator >= 0 ? decoded[..separator] : decoded; + clientId = string.IsNullOrWhiteSpace(clientId) ? null : clientId; + if (clientId is not null) + { + return true; + } + } + } + + return false; + } + + private static bool TryDecodeBase64(ReadOnlySpan encoded, out string decoded) + { + decoded = string.Empty; + + if (encoded.IsEmpty) + { + return false; + } + + try + { + var buffer = Convert.FromBase64String(encoded.ToString()); + decoded = Encoding.UTF8.GetString(buffer); + return true; + } + catch (FormatException) + { + return false; + } + } + + private static class ForwardedHeaderParser + { + public static string? Parse(string headerValue) + { + if (string.IsNullOrWhiteSpace(headerValue)) + { + return null; + } + + var segments = headerValue.Split(';', StringSplitOptions.RemoveEmptyEntries | StringSplitOptions.TrimEntries); + foreach (var segment in segments) + { + var parts = segment.Split('=', 2, StringSplitOptions.TrimEntries); + if (parts.Length != 2) + { + continue; + } + + if (!string.Equals(parts[0], "for", StringComparison.OrdinalIgnoreCase)) + { + continue; + } + + var value = parts[1].Trim('"'); + if (!string.IsNullOrWhiteSpace(value)) + { + return value; + } + } + + return null; + } + } +} diff --git a/src/StellaOps.Authority/StellaOps.Authority/RateLimiting/AuthorityRateLimiterPartitionKeyResolver.cs b/src/StellaOps.Authority/StellaOps.Authority/RateLimiting/AuthorityRateLimiterPartitionKeyResolver.cs new file mode 100644 index 00000000..b71c0df1 --- /dev/null +++ b/src/StellaOps.Authority/StellaOps.Authority/RateLimiting/AuthorityRateLimiterPartitionKeyResolver.cs @@ -0,0 +1,64 @@ +using System; +using System.Net; +using Microsoft.AspNetCore.Http; + +namespace StellaOps.Authority.RateLimiting; + +internal interface IAuthorityRateLimiterPartitionKeyResolver +{ + string Resolve(string limiterName, HttpContext context, AuthorityRateLimiterMetadata metadata); +} + +internal sealed class DefaultAuthorityRateLimiterPartitionKeyResolver : IAuthorityRateLimiterPartitionKeyResolver +{ + public string Resolve(string limiterName, HttpContext context, AuthorityRateLimiterMetadata metadata) + { + ArgumentNullException.ThrowIfNull(limiterName); + ArgumentNullException.ThrowIfNull(context); + ArgumentNullException.ThrowIfNull(metadata); + + var remoteIp = ResolveRemoteIp(context, metadata); + metadata.RemoteIp = remoteIp; + + if ((string.Equals(limiterName, AuthorityRateLimiter.TokenLimiterName, StringComparison.Ordinal) || + string.Equals(limiterName, AuthorityRateLimiter.AuthorizeLimiterName, StringComparison.Ordinal)) && + !string.IsNullOrWhiteSpace(metadata.ClientId)) + { + return $"{limiterName}:{remoteIp}:{metadata.ClientId}"; + } + + return $"{limiterName}:{remoteIp}"; + } + + private static string ResolveRemoteIp(HttpContext context, AuthorityRateLimiterMetadata metadata) + { + if (!string.IsNullOrWhiteSpace(metadata.RemoteIp)) + { + return metadata.RemoteIp!; + } + + if (!string.IsNullOrWhiteSpace(metadata.ForwardedFor)) + { + return metadata.ForwardedFor!; + } + + var normalized = NormalizeRemoteIp(context.Connection.RemoteIpAddress); + return string.IsNullOrWhiteSpace(normalized) ? "unknown" : normalized!; + } + + private static string? NormalizeRemoteIp(IPAddress? address) + { + if (address is null) + { + return null; + } + + if (IPAddress.Any.Equals(address) || IPAddress.IPv6Any.Equals(address) || + IPAddress.IPv6None.Equals(address) || IPAddress.None.Equals(address)) + { + return null; + } + + return address.ToString(); + } +} diff --git a/src/StellaOps.Authority/StellaOps.Authority/RateLimiting/AuthorityRateLimitingApplicationBuilderExtensions.cs b/src/StellaOps.Authority/StellaOps.Authority/RateLimiting/AuthorityRateLimitingApplicationBuilderExtensions.cs new file mode 100644 index 00000000..5c35e4ad --- /dev/null +++ b/src/StellaOps.Authority/StellaOps.Authority/RateLimiting/AuthorityRateLimitingApplicationBuilderExtensions.cs @@ -0,0 +1,13 @@ +using System; +using Microsoft.AspNetCore.Builder; + +namespace StellaOps.Authority.RateLimiting; + +internal static class AuthorityRateLimitingApplicationBuilderExtensions +{ + public static IApplicationBuilder UseAuthorityRateLimiterContext(this IApplicationBuilder app) + { + ArgumentNullException.ThrowIfNull(app); + return app.UseMiddleware(); + } +} diff --git a/src/StellaOps.Authority/StellaOps.Authority/StellaOps.Authority.csproj b/src/StellaOps.Authority/StellaOps.Authority/StellaOps.Authority.csproj new file mode 100644 index 00000000..85fa23fd --- /dev/null +++ b/src/StellaOps.Authority/StellaOps.Authority/StellaOps.Authority.csproj @@ -0,0 +1,28 @@ + + + net10.0 + preview + enable + enable + true + + + + + + + + + + + + + + + + + + + + + diff --git a/src/StellaOps.Authority/StellaOps.Authority/appsettings.Development.json b/src/StellaOps.Authority/StellaOps.Authority/appsettings.Development.json new file mode 100644 index 00000000..ff66ba6b --- /dev/null +++ b/src/StellaOps.Authority/StellaOps.Authority/appsettings.Development.json @@ -0,0 +1,8 @@ +{ + "Logging": { + "LogLevel": { + "Default": "Information", + "Microsoft.AspNetCore": "Warning" + } + } +} diff --git a/src/StellaOps.Authority/StellaOps.Authority/appsettings.json b/src/StellaOps.Authority/StellaOps.Authority/appsettings.json new file mode 100644 index 00000000..4d566948 --- /dev/null +++ b/src/StellaOps.Authority/StellaOps.Authority/appsettings.json @@ -0,0 +1,9 @@ +{ + "Logging": { + "LogLevel": { + "Default": "Information", + "Microsoft.AspNetCore": "Warning" + } + }, + "AllowedHosts": "*" +} diff --git a/src/StellaOps.Authority/TASKS.md b/src/StellaOps.Authority/TASKS.md new file mode 100644 index 00000000..c9309540 --- /dev/null +++ b/src/StellaOps.Authority/TASKS.md @@ -0,0 +1,16 @@ +# Authority Host Task Board (UTC 2025-10-10) + +| ID | Status | Owner(s) | Depends on | Description | Exit Criteria | +|----|--------|----------|------------|-------------|---------------| +| CORE5B.DOC | TODO | Authority Core, Docs Guild | CORE5 | Document token persistence, revocation semantics, and enrichment expectations for resource servers/plugins. | ✅ `docs/11_AUTHORITY.md` + plugin guide updated with claims + token store notes; ✅ Samples include revocation sync guidance. | +| CORE9.REVOCATION | TODO | Authority Core, Security Guild | CORE5 | Implement revocation list persistence + export hooks (API + CLI). | ✅ Revoked tokens denied; ✅ Export endpoint/CLI returns manifest; ✅ Tests cover offline bundle flow. | +| CORE10.JWKS | TODO | Authority Core, DevOps | CORE9.REVOCATION | Provide JWKS rotation with pluggable key loader + documentation. | ✅ Signing/encryption keys rotate without downtime; ✅ JWKS endpoint updates; ✅ Docs describe rotation SOP. | +| CORE8.RL | BLOCKED (Team 2) | Authority Core | CORE8 | Deliver ASP.NET rate limiter plumbing (request metadata, dependency injection hooks) needed by Security Guild. | ✅ `/token` & `/authorize` pipelines expose limiter hooks; ✅ Tests cover throttle behaviour baseline. | +| SEC2.HOST | TODO | Security Guild, Authority Core | SEC2.A (audit contract) | Hook audit logger into OpenIddict handlers and bootstrap endpoints. | ✅ Audit events populated with correlationId, IP, client_id; ✅ Mongo login attempts persisted; ✅ Tests verify on success/failure/lockout. | +| SEC3.HOST | DONE (2025-10-11) | Security Guild | CORE8.RL, SEC3.A (rate policy) | Apply rate limiter policies (`AddRateLimiter`) to `/token` and `/internal/*` endpoints with configuration binding. | ✅ Policies configurable via `StellaOpsAuthorityOptions.Security.RateLimiting`; ✅ Integration tests hit 429 after limit; ✅ Docs updated. | +| SEC4.HOST | TODO | Security Guild, DevOps | SEC4.A (revocation schema) | Implement CLI/HTTP surface to export revocation bundle + detached JWS using `StellaOps.Cryptography`. | ✅ `stellaops auth revoke export` CLI/endpoint returns JSON + `.jws`; ✅ Verification script passes; ✅ Operator docs updated. | +| SEC4.KEY | TODO | Security Guild, DevOps | SEC4.HOST | Integrate signing keys with provider registry (initial ES256). | ✅ Keys loaded via `ICryptoProvider` signer; ✅ Rotation SOP documented. | +| SEC5.HOST | TODO | Security Guild | SEC5.A (threat model) | Feed Authority-specific mitigations (rate limiting, audit, revocation) into threat model + backlog. | ✅ Threat model updated; ✅ Backlog issues reference mitigations; ✅ Review sign-off captured. | +| SEC3.BUILD | DONE (2025-10-11) | Authority Core, Security Guild | SEC3.HOST, FEEDMERGE-COORD-02-900 | Track normalized-range dependency fallout and restore full test matrix once Feedser range primitives land. | ✅ Feedser normalized range libraries merged; ✅ Authority + Configuration test suites (`dotnet test src/StellaOps.Authority.sln`, `dotnet test src/StellaOps.Configuration.Tests/StellaOps.Configuration.Tests.csproj`) pass without Feedser compile failures; ✅ Status recorded here/Sprints (authority-core broadcast not available). | + +> Update status columns (TODO / DOING / DONE / BLOCKED) together with code changes. Always run `dotnet test src/StellaOps.Authority.sln` when touching host logic. diff --git a/src/StellaOps.Cli.Tests/Commands/CommandHandlersTests.cs b/src/StellaOps.Cli.Tests/Commands/CommandHandlersTests.cs new file mode 100644 index 00000000..c7bfdce4 --- /dev/null +++ b/src/StellaOps.Cli.Tests/Commands/CommandHandlersTests.cs @@ -0,0 +1,533 @@ +using System; +using System.Collections.Generic; +using System.IO; +using System.Text; +using System.Text.Json; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Logging; +using Microsoft.IdentityModel.Tokens; +using StellaOps.Auth.Abstractions; +using StellaOps.Auth.Client; +using StellaOps.Cli.Commands; +using StellaOps.Cli.Configuration; +using StellaOps.Cli.Services; +using StellaOps.Cli.Services.Models; +using StellaOps.Cli.Telemetry; +using StellaOps.Cli.Tests.Testing; + +namespace StellaOps.Cli.Tests.Commands; + +public sealed class CommandHandlersTests +{ + [Fact] + public async Task HandleExportJobAsync_SetsExitCodeZeroOnSuccess() + { + var original = Environment.ExitCode; + try + { + var backend = new StubBackendClient(new JobTriggerResult(true, "Accepted", "/jobs/export:json/1", null)); + var provider = BuildServiceProvider(backend); + + await CommandHandlers.HandleExportJobAsync( + provider, + format: "json", + delta: false, + publishFull: null, + publishDelta: null, + includeFull: null, + includeDelta: null, + verbose: false, + cancellationToken: CancellationToken.None); + + Assert.Equal(0, Environment.ExitCode); + Assert.Equal("export:json", backend.LastJobKind); + } + finally + { + Environment.ExitCode = original; + } + } + + [Fact] + public async Task HandleMergeJobAsync_SetsExitCodeOnFailure() + { + var original = Environment.ExitCode; + try + { + var backend = new StubBackendClient(new JobTriggerResult(false, "Job already running", null, null)); + var provider = BuildServiceProvider(backend); + + await CommandHandlers.HandleMergeJobAsync(provider, verbose: false, CancellationToken.None); + + Assert.Equal(1, Environment.ExitCode); + Assert.Equal("merge:reconcile", backend.LastJobKind); + } + finally + { + Environment.ExitCode = original; + } + } + + [Fact] + public async Task HandleScannerRunAsync_AutomaticallyUploadsResults() + { + using var tempDir = new TempDirectory(); + var resultsFile = Path.Combine(tempDir.Path, "results", "scan.json"); + var backend = new StubBackendClient(new JobTriggerResult(true, "Accepted", null, null)); + var metadataFile = Path.Combine(tempDir.Path, "results", "scan-run.json"); + var executor = new StubExecutor(new ScannerExecutionResult(0, resultsFile, metadataFile)); + var options = new StellaOpsCliOptions + { + ResultsDirectory = Path.Combine(tempDir.Path, "results") + }; + + var provider = BuildServiceProvider(backend, executor, new StubInstaller(), options); + + Directory.CreateDirectory(Path.Combine(tempDir.Path, "target")); + + var original = Environment.ExitCode; + try + { + await CommandHandlers.HandleScannerRunAsync( + provider, + runner: "docker", + entry: "scanner-image", + targetDirectory: Path.Combine(tempDir.Path, "target"), + arguments: Array.Empty(), + verbose: false, + cancellationToken: CancellationToken.None); + + Assert.Equal(0, Environment.ExitCode); + Assert.Equal(resultsFile, backend.LastUploadPath); + Assert.True(File.Exists(metadataFile)); + } + finally + { + Environment.ExitCode = original; + } + } + + [Fact] + public async Task HandleAuthLoginAsync_UsesClientCredentialsFlow() + { + var original = Environment.ExitCode; + using var tempDir = new TempDirectory(); + + try + { + var options = new StellaOpsCliOptions + { + ResultsDirectory = Path.Combine(tempDir.Path, "results"), + Authority = new StellaOpsCliAuthorityOptions + { + Url = "https://authority.example", + ClientId = "cli", + ClientSecret = "secret", + Scope = "feedser.jobs.trigger", + TokenCacheDirectory = tempDir.Path + } + }; + + var tokenClient = new StubTokenClient(); + var provider = BuildServiceProvider(new StubBackendClient(new JobTriggerResult(true, "ok", null, null)), options: options, tokenClient: tokenClient); + + await CommandHandlers.HandleAuthLoginAsync(provider, options, verbose: false, force: false, cancellationToken: CancellationToken.None); + + Assert.Equal(0, Environment.ExitCode); + Assert.Equal(1, tokenClient.ClientCredentialRequests); + Assert.NotNull(tokenClient.CachedEntry); + } + finally + { + Environment.ExitCode = original; + } + } + + [Fact] + public async Task HandleAuthLoginAsync_FailsWhenPasswordMissing() + { + var original = Environment.ExitCode; + using var tempDir = new TempDirectory(); + + try + { + var options = new StellaOpsCliOptions + { + ResultsDirectory = Path.Combine(tempDir.Path, "results"), + Authority = new StellaOpsCliAuthorityOptions + { + Url = "https://authority.example", + ClientId = "cli", + Username = "user", + TokenCacheDirectory = tempDir.Path + } + }; + + var provider = BuildServiceProvider(new StubBackendClient(new JobTriggerResult(true, "ok", null, null)), options: options, tokenClient: new StubTokenClient()); + + await CommandHandlers.HandleAuthLoginAsync(provider, options, verbose: false, force: false, cancellationToken: CancellationToken.None); + + Assert.Equal(1, Environment.ExitCode); + } + finally + { + Environment.ExitCode = original; + } + } + + [Fact] + public async Task HandleAuthStatusAsync_ReportsMissingToken() + { + var original = Environment.ExitCode; + using var tempDir = new TempDirectory(); + + try + { + var options = new StellaOpsCliOptions + { + ResultsDirectory = Path.Combine(tempDir.Path, "results"), + Authority = new StellaOpsCliAuthorityOptions + { + Url = "https://authority.example", + ClientId = "cli", + TokenCacheDirectory = tempDir.Path + } + }; + + var provider = BuildServiceProvider(new StubBackendClient(new JobTriggerResult(true, "ok", null, null)), options: options, tokenClient: new StubTokenClient()); + + await CommandHandlers.HandleAuthStatusAsync(provider, options, verbose: false, cancellationToken: CancellationToken.None); + + Assert.Equal(1, Environment.ExitCode); + } + finally + { + Environment.ExitCode = original; + } + } + + [Fact] + public async Task HandleAuthStatusAsync_ReportsCachedToken() + { + var original = Environment.ExitCode; + using var tempDir = new TempDirectory(); + + try + { + var options = new StellaOpsCliOptions + { + ResultsDirectory = Path.Combine(tempDir.Path, "results"), + Authority = new StellaOpsCliAuthorityOptions + { + Url = "https://authority.example", + ClientId = "cli", + TokenCacheDirectory = tempDir.Path + } + }; + + var tokenClient = new StubTokenClient(); + tokenClient.CachedEntry = new StellaOpsTokenCacheEntry( + "token", + "Bearer", + DateTimeOffset.UtcNow.AddMinutes(30), + new[] { StellaOpsScopes.FeedserJobsTrigger }); + + var provider = BuildServiceProvider(new StubBackendClient(new JobTriggerResult(true, "ok", null, null)), options: options, tokenClient: tokenClient); + + await CommandHandlers.HandleAuthStatusAsync(provider, options, verbose: true, cancellationToken: CancellationToken.None); + + Assert.Equal(0, Environment.ExitCode); + } + finally + { + Environment.ExitCode = original; + } + } + + [Fact] + public async Task HandleAuthWhoAmIAsync_ReturnsErrorWhenTokenMissing() + { + var original = Environment.ExitCode; + using var tempDir = new TempDirectory(); + + try + { + var options = new StellaOpsCliOptions + { + ResultsDirectory = Path.Combine(tempDir.Path, "results"), + Authority = new StellaOpsCliAuthorityOptions + { + Url = "https://authority.example", + ClientId = "cli", + TokenCacheDirectory = tempDir.Path + } + }; + + var provider = BuildServiceProvider(new StubBackendClient(new JobTriggerResult(true, "ok", null, null)), options: options, tokenClient: new StubTokenClient()); + + await CommandHandlers.HandleAuthWhoAmIAsync(provider, options, verbose: false, cancellationToken: CancellationToken.None); + + Assert.Equal(1, Environment.ExitCode); + } + finally + { + Environment.ExitCode = original; + } + } + + [Fact] + public async Task HandleAuthWhoAmIAsync_ReportsClaimsForJwtToken() + { + var original = Environment.ExitCode; + using var tempDir = new TempDirectory(); + + try + { + var options = new StellaOpsCliOptions + { + ResultsDirectory = Path.Combine(tempDir.Path, "results"), + Authority = new StellaOpsCliAuthorityOptions + { + Url = "https://authority.example", + ClientId = "cli", + TokenCacheDirectory = tempDir.Path + } + }; + + var tokenClient = new StubTokenClient(); + tokenClient.CachedEntry = new StellaOpsTokenCacheEntry( + CreateUnsignedJwt( + ("sub", "cli-user"), + ("aud", "feedser"), + ("iss", "https://authority.example"), + ("iat", 1_700_000_000), + ("nbf", 1_700_000_000)), + "Bearer", + DateTimeOffset.UtcNow.AddMinutes(30), + new[] { StellaOpsScopes.FeedserJobsTrigger }); + + var provider = BuildServiceProvider(new StubBackendClient(new JobTriggerResult(true, "ok", null, null)), options: options, tokenClient: tokenClient); + + await CommandHandlers.HandleAuthWhoAmIAsync(provider, options, verbose: true, cancellationToken: CancellationToken.None); + + Assert.Equal(0, Environment.ExitCode); + } + finally + { + Environment.ExitCode = original; + } + } + + [Fact] + public async Task HandleAuthLogoutAsync_ClearsToken() + { + var original = Environment.ExitCode; + using var tempDir = new TempDirectory(); + + try + { + var options = new StellaOpsCliOptions + { + ResultsDirectory = Path.Combine(tempDir.Path, "results"), + Authority = new StellaOpsCliAuthorityOptions + { + Url = "https://authority.example", + ClientId = "cli", + TokenCacheDirectory = tempDir.Path + } + }; + + var tokenClient = new StubTokenClient(); + tokenClient.CachedEntry = new StellaOpsTokenCacheEntry( + "token", + "Bearer", + DateTimeOffset.UtcNow.AddMinutes(5), + new[] { StellaOpsScopes.FeedserJobsTrigger }); + + var provider = BuildServiceProvider(new StubBackendClient(new JobTriggerResult(true, "ok", null, null)), options: options, tokenClient: tokenClient); + + await CommandHandlers.HandleAuthLogoutAsync(provider, options, verbose: true, cancellationToken: CancellationToken.None); + + Assert.Null(tokenClient.CachedEntry); + Assert.Equal(1, tokenClient.ClearRequests); + Assert.Equal(0, Environment.ExitCode); + } + finally + { + Environment.ExitCode = original; + } + } + + private static IServiceProvider BuildServiceProvider( + IBackendOperationsClient backend, + IScannerExecutor? executor = null, + IScannerInstaller? installer = null, + StellaOpsCliOptions? options = null, + IStellaOpsTokenClient? tokenClient = null) + { + var services = new ServiceCollection(); + services.AddSingleton(backend); + services.AddSingleton(_ => LoggerFactory.Create(builder => builder.SetMinimumLevel(LogLevel.Debug))); + services.AddSingleton(new VerbosityState()); + var resolvedOptions = options ?? new StellaOpsCliOptions + { + ResultsDirectory = Path.Combine(Path.GetTempPath(), $"stellaops-cli-results-{Guid.NewGuid():N}") + }; + services.AddSingleton(resolvedOptions); + + var resolvedExecutor = executor ?? CreateDefaultExecutor(); + services.AddSingleton(resolvedExecutor); + services.AddSingleton(installer ?? new StubInstaller()); + + if (tokenClient is not null) + { + services.AddSingleton(tokenClient); + } + + return services.BuildServiceProvider(); + } + + private static IScannerExecutor CreateDefaultExecutor() + { + var tempResultsFile = Path.GetTempFileName(); + var tempMetadataFile = Path.Combine( + Path.GetDirectoryName(tempResultsFile)!, + $"{Path.GetFileNameWithoutExtension(tempResultsFile)}-run.json"); + return new StubExecutor(new ScannerExecutionResult(0, tempResultsFile, tempMetadataFile)); + } + + private sealed class StubBackendClient : IBackendOperationsClient + { + private readonly JobTriggerResult _result; + + public StubBackendClient(JobTriggerResult result) + { + _result = result; + } + + public string? LastJobKind { get; private set; } + public string? LastUploadPath { get; private set; } + + public Task DownloadScannerAsync(string channel, string outputPath, bool overwrite, bool verbose, CancellationToken cancellationToken) + => throw new NotImplementedException(); + + public Task UploadScanResultsAsync(string filePath, CancellationToken cancellationToken) + { + LastUploadPath = filePath; + return Task.CompletedTask; + } + + public Task TriggerJobAsync(string jobKind, IDictionary parameters, CancellationToken cancellationToken) + { + LastJobKind = jobKind; + return Task.FromResult(_result); + } + } + + private sealed class StubExecutor : IScannerExecutor + { + private readonly ScannerExecutionResult _result; + + public StubExecutor(ScannerExecutionResult result) + { + _result = result; + } + + public Task RunAsync(string runner, string entry, string targetDirectory, string resultsDirectory, IReadOnlyList arguments, bool verbose, CancellationToken cancellationToken) + { + Directory.CreateDirectory(Path.GetDirectoryName(_result.ResultsPath)!); + if (!File.Exists(_result.ResultsPath)) + { + File.WriteAllText(_result.ResultsPath, "{}"); + } + + Directory.CreateDirectory(Path.GetDirectoryName(_result.RunMetadataPath)!); + if (!File.Exists(_result.RunMetadataPath)) + { + File.WriteAllText(_result.RunMetadataPath, "{}"); + } + + return Task.FromResult(_result); + } + } + + private sealed class StubInstaller : IScannerInstaller + { + public Task InstallAsync(string artifactPath, bool verbose, CancellationToken cancellationToken) + => Task.CompletedTask; + } + + private sealed class StubTokenClient : IStellaOpsTokenClient + { + private readonly StellaOpsTokenResult _token; + + public StubTokenClient() + { + _token = new StellaOpsTokenResult( + "token-123", + "Bearer", + DateTimeOffset.UtcNow.AddMinutes(30), + new[] { StellaOpsScopes.FeedserJobsTrigger }); + } + + public int ClientCredentialRequests { get; private set; } + public int PasswordRequests { get; private set; } + public int ClearRequests { get; private set; } + public StellaOpsTokenCacheEntry? CachedEntry { get; set; } + + public ValueTask CacheTokenAsync(string key, StellaOpsTokenCacheEntry entry, CancellationToken cancellationToken = default) + { + CachedEntry = entry; + return ValueTask.CompletedTask; + } + + public ValueTask ClearCachedTokenAsync(string key, CancellationToken cancellationToken = default) + { + ClearRequests++; + CachedEntry = null; + return ValueTask.CompletedTask; + } + + public Task GetJsonWebKeySetAsync(CancellationToken cancellationToken = default) + => Task.FromResult(new JsonWebKeySet("{\"keys\":[]}")); + + public ValueTask GetCachedTokenAsync(string key, CancellationToken cancellationToken = default) + => ValueTask.FromResult(CachedEntry); + + public Task RequestClientCredentialsTokenAsync(string? scope = null, CancellationToken cancellationToken = default) + { + ClientCredentialRequests++; + return Task.FromResult(_token); + } + + public Task RequestPasswordTokenAsync(string username, string password, string? scope = null, CancellationToken cancellationToken = default) + { + PasswordRequests++; + return Task.FromResult(_token); + } + } + + private static string CreateUnsignedJwt(params (string Key, object Value)[] claims) + { + var headerJson = "{\"alg\":\"none\",\"typ\":\"JWT\"}"; + var payload = new Dictionary(StringComparer.Ordinal); + foreach (var claim in claims) + { + payload[claim.Key] = claim.Value; + } + + var payloadJson = JsonSerializer.Serialize(payload); + return $"{Base64UrlEncode(headerJson)}.{Base64UrlEncode(payloadJson)}."; + } + + private static string Base64UrlEncode(string value) + { + var bytes = Encoding.UTF8.GetBytes(value); + return Convert.ToBase64String(bytes) + .TrimEnd('=') + .Replace('+', '-') + .Replace('/', '_'); + } +} diff --git a/src/StellaOps.Cli.Tests/Configuration/CliBootstrapperTests.cs b/src/StellaOps.Cli.Tests/Configuration/CliBootstrapperTests.cs new file mode 100644 index 00000000..d19deb4f --- /dev/null +++ b/src/StellaOps.Cli.Tests/Configuration/CliBootstrapperTests.cs @@ -0,0 +1,110 @@ +using System; +using System.IO; +using System.Text.Json; +using StellaOps.Cli.Configuration; +using Xunit; + +namespace StellaOps.Cli.Tests.Configuration; + +public sealed class CliBootstrapperTests : IDisposable +{ + private readonly string _originalDirectory = Directory.GetCurrentDirectory(); + private readonly string _tempDirectory = Path.Combine(Path.GetTempPath(), $"stellaops-cli-tests-{Guid.NewGuid():N}"); + + public CliBootstrapperTests() + { + Directory.CreateDirectory(_tempDirectory); + Directory.SetCurrentDirectory(_tempDirectory); + } + + [Fact] + public void Build_UsesEnvironmentVariablesWhenPresent() + { + Environment.SetEnvironmentVariable("API_KEY", "env-key"); + Environment.SetEnvironmentVariable("STELLAOPS_BACKEND_URL", "https://env-backend.example"); + Environment.SetEnvironmentVariable("STELLAOPS_AUTHORITY_URL", "https://authority.env"); + Environment.SetEnvironmentVariable("STELLAOPS_AUTHORITY_CLIENT_ID", "cli-env"); + Environment.SetEnvironmentVariable("STELLAOPS_AUTHORITY_SCOPE", "feedser.jobs.trigger"); + Environment.SetEnvironmentVariable("STELLAOPS_AUTHORITY_ENABLE_RETRIES", "false"); + Environment.SetEnvironmentVariable("STELLAOPS_AUTHORITY_RETRY_DELAYS", "00:00:02,00:00:05"); + Environment.SetEnvironmentVariable("STELLAOPS_AUTHORITY_ALLOW_OFFLINE_CACHE_FALLBACK", "false"); + Environment.SetEnvironmentVariable("STELLAOPS_AUTHORITY_OFFLINE_CACHE_TOLERANCE", "00:20:00"); + + try + { + var (options, _) = CliBootstrapper.Build(Array.Empty()); + + Assert.Equal("env-key", options.ApiKey); + Assert.Equal("https://env-backend.example", options.BackendUrl); + Assert.Equal("https://authority.env", options.Authority.Url); + Assert.Equal("cli-env", options.Authority.ClientId); + Assert.Equal("feedser.jobs.trigger", options.Authority.Scope); + + Assert.NotNull(options.Authority.Resilience); + Assert.False(options.Authority.Resilience.EnableRetries); + Assert.Equal(new[] { TimeSpan.FromSeconds(2), TimeSpan.FromSeconds(5) }, options.Authority.Resilience.RetryDelays); + Assert.False(options.Authority.Resilience.AllowOfflineCacheFallback); + Assert.Equal(TimeSpan.FromMinutes(20), options.Authority.Resilience.OfflineCacheTolerance); + } + finally + { + Environment.SetEnvironmentVariable("API_KEY", null); + Environment.SetEnvironmentVariable("STELLAOPS_BACKEND_URL", null); + Environment.SetEnvironmentVariable("STELLAOPS_AUTHORITY_URL", null); + Environment.SetEnvironmentVariable("STELLAOPS_AUTHORITY_CLIENT_ID", null); + Environment.SetEnvironmentVariable("STELLAOPS_AUTHORITY_SCOPE", null); + Environment.SetEnvironmentVariable("STELLAOPS_AUTHORITY_ENABLE_RETRIES", null); + Environment.SetEnvironmentVariable("STELLAOPS_AUTHORITY_RETRY_DELAYS", null); + Environment.SetEnvironmentVariable("STELLAOPS_AUTHORITY_ALLOW_OFFLINE_CACHE_FALLBACK", null); + Environment.SetEnvironmentVariable("STELLAOPS_AUTHORITY_OFFLINE_CACHE_TOLERANCE", null); + } + } + + [Fact] + public void Build_FallsBackToAppSettings() + { + WriteAppSettings(new + { + StellaOps = new + { + ApiKey = "file-key", + BackendUrl = "https://file-backend.example", + Authority = new + { + Url = "https://authority.file", + ClientId = "cli-file", + Scope = "feedser.jobs.trigger" + } + } + }); + + var (options, _) = CliBootstrapper.Build(Array.Empty()); + + Assert.Equal("file-key", options.ApiKey); + Assert.Equal("https://file-backend.example", options.BackendUrl); + Assert.Equal("https://authority.file", options.Authority.Url); + Assert.Equal("cli-file", options.Authority.ClientId); + } + + public void Dispose() + { + Directory.SetCurrentDirectory(_originalDirectory); + if (Directory.Exists(_tempDirectory)) + { + try + { + Directory.Delete(_tempDirectory, recursive: true); + } + catch + { + // Ignored. + } + } + } + + private static void WriteAppSettings(T payload) + { + var json = JsonSerializer.Serialize(payload, new JsonSerializerOptions { WriteIndented = true }); + File.WriteAllText("appsettings.json", json); + } +} diff --git a/src/StellaOps.Cli.Tests/Services/BackendOperationsClientTests.cs b/src/StellaOps.Cli.Tests/Services/BackendOperationsClientTests.cs new file mode 100644 index 00000000..14587016 --- /dev/null +++ b/src/StellaOps.Cli.Tests/Services/BackendOperationsClientTests.cs @@ -0,0 +1,417 @@ +using System; +using System.IO; +using System.Net; +using System.Net.Http; +using System.Net.Http.Json; +using System.Security.Cryptography; +using System.Text; +using System.Text.Json; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.Extensions.Logging; +using Microsoft.IdentityModel.Tokens; +using StellaOps.Auth.Abstractions; +using StellaOps.Auth.Client; +using StellaOps.Cli.Configuration; +using StellaOps.Cli.Services; +using StellaOps.Cli.Services.Models; +using StellaOps.Cli.Services.Models.Transport; +using StellaOps.Cli.Tests.Testing; + +namespace StellaOps.Cli.Tests.Services; + +public sealed class BackendOperationsClientTests +{ + [Fact] + public async Task DownloadScannerAsync_VerifiesDigestAndWritesMetadata() + { + using var temp = new TempDirectory(); + + var contentBytes = Encoding.UTF8.GetBytes("scanner-blob"); + var digestHex = Convert.ToHexString(SHA256.HashData(contentBytes)).ToLowerInvariant(); + + var handler = new StubHttpMessageHandler((request, _) => + { + var response = new HttpResponseMessage(HttpStatusCode.OK) + { + Content = new ByteArrayContent(contentBytes), + RequestMessage = request + }; + + response.Headers.Add("X-StellaOps-Digest", $"sha256:{digestHex}"); + response.Content.Headers.LastModified = DateTimeOffset.UtcNow; + response.Content.Headers.ContentType = new System.Net.Http.Headers.MediaTypeHeaderValue("application/octet-stream"); + return response; + }); + + var httpClient = new HttpClient(handler) + { + BaseAddress = new Uri("https://feedser.example") + }; + + var options = new StellaOpsCliOptions + { + BackendUrl = "https://feedser.example", + ScannerCacheDirectory = temp.Path, + ScannerDownloadAttempts = 1 + }; + + var loggerFactory = LoggerFactory.Create(builder => builder.SetMinimumLevel(LogLevel.Debug)); + var client = new BackendOperationsClient(httpClient, options, loggerFactory.CreateLogger()); + + var targetPath = Path.Combine(temp.Path, "scanner.tar.gz"); + var result = await client.DownloadScannerAsync("stable", targetPath, overwrite: false, verbose: true, CancellationToken.None); + + Assert.False(result.FromCache); + Assert.True(File.Exists(targetPath)); + + var metadataPath = targetPath + ".metadata.json"; + Assert.True(File.Exists(metadataPath)); + + using var document = JsonDocument.Parse(File.ReadAllText(metadataPath)); + Assert.Equal($"sha256:{digestHex}", document.RootElement.GetProperty("digest").GetString()); + Assert.Equal("stable", document.RootElement.GetProperty("channel").GetString()); + } + + [Fact] + public async Task DownloadScannerAsync_ThrowsOnDigestMismatch() + { + using var temp = new TempDirectory(); + + var contentBytes = Encoding.UTF8.GetBytes("scanner-data"); + var handler = new StubHttpMessageHandler((request, _) => + { + var response = new HttpResponseMessage(HttpStatusCode.OK) + { + Content = new ByteArrayContent(contentBytes), + RequestMessage = request + }; + response.Headers.Add("X-StellaOps-Digest", "sha256:deadbeef"); + return response; + }); + + var httpClient = new HttpClient(handler) + { + BaseAddress = new Uri("https://feedser.example") + }; + + var options = new StellaOpsCliOptions + { + BackendUrl = "https://feedser.example", + ScannerCacheDirectory = temp.Path, + ScannerDownloadAttempts = 1 + }; + + var loggerFactory = LoggerFactory.Create(builder => builder.SetMinimumLevel(LogLevel.Debug)); + var client = new BackendOperationsClient(httpClient, options, loggerFactory.CreateLogger()); + + var targetPath = Path.Combine(temp.Path, "scanner.tar.gz"); + + await Assert.ThrowsAsync(() => client.DownloadScannerAsync("stable", targetPath, overwrite: true, verbose: false, CancellationToken.None)); + Assert.False(File.Exists(targetPath)); + } + + [Fact] + public async Task DownloadScannerAsync_RetriesOnFailure() + { + using var temp = new TempDirectory(); + + var successBytes = Encoding.UTF8.GetBytes("success"); + var digestHex = Convert.ToHexString(SHA256.HashData(successBytes)).ToLowerInvariant(); + var attempts = 0; + + var handler = new StubHttpMessageHandler( + (request, _) => + { + attempts++; + return new HttpResponseMessage(HttpStatusCode.InternalServerError) + { + RequestMessage = request, + Content = new StringContent("error") + }; + }, + (request, _) => + { + attempts++; + var response = new HttpResponseMessage(HttpStatusCode.OK) + { + RequestMessage = request, + Content = new ByteArrayContent(successBytes) + }; + response.Headers.Add("X-StellaOps-Digest", $"sha256:{digestHex}"); + return response; + }); + + var httpClient = new HttpClient(handler) + { + BaseAddress = new Uri("https://feedser.example") + }; + + var options = new StellaOpsCliOptions + { + BackendUrl = "https://feedser.example", + ScannerCacheDirectory = temp.Path, + ScannerDownloadAttempts = 3 + }; + + var loggerFactory = LoggerFactory.Create(builder => builder.SetMinimumLevel(LogLevel.Debug)); + var client = new BackendOperationsClient(httpClient, options, loggerFactory.CreateLogger()); + + var targetPath = Path.Combine(temp.Path, "scanner.tar.gz"); + var result = await client.DownloadScannerAsync("stable", targetPath, overwrite: false, verbose: false, CancellationToken.None); + + Assert.Equal(2, attempts); + Assert.False(result.FromCache); + Assert.True(File.Exists(targetPath)); + } + + [Fact] + public async Task UploadScanResultsAsync_RetriesOnRetryAfter() + { + using var temp = new TempDirectory(); + var filePath = Path.Combine(temp.Path, "scan.json"); + await File.WriteAllTextAsync(filePath, "{}"); + + var attempts = 0; + var handler = new StubHttpMessageHandler( + (request, _) => + { + attempts++; + var response = new HttpResponseMessage(HttpStatusCode.TooManyRequests) + { + RequestMessage = request, + Content = new StringContent("busy") + }; + response.Headers.Add("Retry-After", "1"); + return response; + }, + (request, _) => + { + attempts++; + return new HttpResponseMessage(HttpStatusCode.OK) + { + RequestMessage = request + }; + }); + + var httpClient = new HttpClient(handler) + { + BaseAddress = new Uri("https://feedser.example") + }; + + var options = new StellaOpsCliOptions + { + BackendUrl = "https://feedser.example", + ScanUploadAttempts = 3 + }; + + var loggerFactory = LoggerFactory.Create(builder => builder.SetMinimumLevel(LogLevel.Debug)); + var client = new BackendOperationsClient(httpClient, options, loggerFactory.CreateLogger()); + + await client.UploadScanResultsAsync(filePath, CancellationToken.None); + + Assert.Equal(2, attempts); + } + + [Fact] + public async Task UploadScanResultsAsync_ThrowsAfterMaxAttempts() + { + using var temp = new TempDirectory(); + var filePath = Path.Combine(temp.Path, "scan.json"); + await File.WriteAllTextAsync(filePath, "{}"); + + var attempts = 0; + var handler = new StubHttpMessageHandler( + (request, _) => + { + attempts++; + return new HttpResponseMessage(HttpStatusCode.BadGateway) + { + RequestMessage = request, + Content = new StringContent("bad gateway") + }; + }); + + var httpClient = new HttpClient(handler) + { + BaseAddress = new Uri("https://feedser.example") + }; + + var options = new StellaOpsCliOptions + { + BackendUrl = "https://feedser.example", + ScanUploadAttempts = 2 + }; + + var loggerFactory = LoggerFactory.Create(builder => builder.SetMinimumLevel(LogLevel.Debug)); + var client = new BackendOperationsClient(httpClient, options, loggerFactory.CreateLogger()); + + await Assert.ThrowsAsync(() => client.UploadScanResultsAsync(filePath, CancellationToken.None)); + Assert.Equal(2, attempts); + } + + [Fact] + public async Task TriggerJobAsync_ReturnsAcceptedResult() + { + var handler = new StubHttpMessageHandler((request, _) => + { + var response = new HttpResponseMessage(HttpStatusCode.Accepted) + { + RequestMessage = request, + Content = JsonContent.Create(new JobRunResponse + { + RunId = Guid.NewGuid(), + Status = "queued", + Kind = "export:json", + Trigger = "cli", + CreatedAt = DateTimeOffset.UtcNow + }) + }; + response.Headers.Location = new Uri("/jobs/export:json/runs/123", UriKind.Relative); + return response; + }); + + var httpClient = new HttpClient(handler) + { + BaseAddress = new Uri("https://feedser.example") + }; + + var options = new StellaOpsCliOptions { BackendUrl = "https://feedser.example" }; + var loggerFactory = LoggerFactory.Create(builder => builder.SetMinimumLevel(LogLevel.Debug)); + var client = new BackendOperationsClient(httpClient, options, loggerFactory.CreateLogger()); + + var result = await client.TriggerJobAsync("export:json", new Dictionary(), CancellationToken.None); + + Assert.True(result.Success); + Assert.Equal("Accepted", result.Message); + Assert.Equal("/jobs/export:json/runs/123", result.Location); + } + + [Fact] + public async Task TriggerJobAsync_ReturnsFailureMessage() + { + var handler = new StubHttpMessageHandler((request, _) => + { + var problem = new + { + title = "Job already running", + detail = "export job active" + }; + + var response = new HttpResponseMessage(HttpStatusCode.Conflict) + { + RequestMessage = request, + Content = JsonContent.Create(problem) + }; + return response; + }); + + var httpClient = new HttpClient(handler) + { + BaseAddress = new Uri("https://feedser.example") + }; + + var options = new StellaOpsCliOptions { BackendUrl = "https://feedser.example" }; + var loggerFactory = LoggerFactory.Create(builder => builder.SetMinimumLevel(LogLevel.Debug)); + var client = new BackendOperationsClient(httpClient, options, loggerFactory.CreateLogger()); + + var result = await client.TriggerJobAsync("export:json", new Dictionary(), CancellationToken.None); + + Assert.False(result.Success); + Assert.Contains("Job already running", result.Message); + } + + [Fact] + public async Task TriggerJobAsync_UsesAuthorityTokenWhenConfigured() + { + using var temp = new TempDirectory(); + + var handler = new StubHttpMessageHandler((request, _) => + { + Assert.NotNull(request.Headers.Authorization); + Assert.Equal("Bearer", request.Headers.Authorization!.Scheme); + Assert.Equal("token-123", request.Headers.Authorization.Parameter); + + return new HttpResponseMessage(HttpStatusCode.Accepted) + { + RequestMessage = request, + Content = JsonContent.Create(new JobRunResponse + { + RunId = Guid.NewGuid(), + Kind = "test", + Status = "Pending", + Trigger = "cli", + CreatedAt = DateTimeOffset.UtcNow + }) + }; + }); + + var httpClient = new HttpClient(handler) + { + BaseAddress = new Uri("https://feedser.example") + }; + + var options = new StellaOpsCliOptions + { + BackendUrl = "https://feedser.example", + Authority = + { + Url = "https://authority.example", + ClientId = "cli", + ClientSecret = "secret", + Scope = "feedser.jobs.trigger", + TokenCacheDirectory = temp.Path + } + }; + + var tokenClient = new StubTokenClient(); + var loggerFactory = LoggerFactory.Create(builder => builder.SetMinimumLevel(LogLevel.Debug)); + var client = new BackendOperationsClient(httpClient, options, loggerFactory.CreateLogger(), tokenClient); + + var result = await client.TriggerJobAsync("test", new Dictionary(), CancellationToken.None); + + Assert.True(result.Success); + Assert.Equal("Accepted", result.Message); + Assert.True(tokenClient.Requests > 0); + } + + private sealed class StubTokenClient : IStellaOpsTokenClient + { + private readonly StellaOpsTokenResult _tokenResult; + + public int Requests { get; private set; } + + public StubTokenClient() + { + _tokenResult = new StellaOpsTokenResult( + "token-123", + "Bearer", + DateTimeOffset.UtcNow.AddMinutes(5), + new[] { StellaOpsScopes.FeedserJobsTrigger }); + } + + public ValueTask CacheTokenAsync(string key, StellaOpsTokenCacheEntry entry, CancellationToken cancellationToken = default) + => ValueTask.CompletedTask; + + public ValueTask ClearCachedTokenAsync(string key, CancellationToken cancellationToken = default) + => ValueTask.CompletedTask; + + public Task GetJsonWebKeySetAsync(CancellationToken cancellationToken = default) + => Task.FromResult(new JsonWebKeySet("{\"keys\":[]}")); + + public ValueTask GetCachedTokenAsync(string key, CancellationToken cancellationToken = default) + => ValueTask.FromResult(null); + + public Task RequestClientCredentialsTokenAsync(string? scope = null, CancellationToken cancellationToken = default) + { + Requests++; + return Task.FromResult(_tokenResult); + } + + public Task RequestPasswordTokenAsync(string username, string password, string? scope = null, CancellationToken cancellationToken = default) + { + Requests++; + return Task.FromResult(_tokenResult); + } + } +} diff --git a/src/StellaOps.Cli.Tests/StellaOps.Cli.Tests.csproj b/src/StellaOps.Cli.Tests/StellaOps.Cli.Tests.csproj new file mode 100644 index 00000000..ab4b3308 --- /dev/null +++ b/src/StellaOps.Cli.Tests/StellaOps.Cli.Tests.csproj @@ -0,0 +1,28 @@ + + + + net10.0 + enable + enable + false + + + + + + + + + + + + + + + + + + + + + diff --git a/src/StellaOps.Cli.Tests/Testing/TestHelpers.cs b/src/StellaOps.Cli.Tests/Testing/TestHelpers.cs new file mode 100644 index 00000000..561e27da --- /dev/null +++ b/src/StellaOps.Cli.Tests/Testing/TestHelpers.cs @@ -0,0 +1,55 @@ +using System; +using System.Collections.Generic; +using System.IO; +using System.Net.Http; +using System.Threading; +using System.Threading.Tasks; + +namespace StellaOps.Cli.Tests.Testing; + +internal sealed class TempDirectory : IDisposable +{ + public TempDirectory() + { + Path = System.IO.Path.Combine(System.IO.Path.GetTempPath(), $"stellaops-cli-tests-{Guid.NewGuid():N}"); + Directory.CreateDirectory(Path); + } + + public string Path { get; } + + public void Dispose() + { + try + { + if (Directory.Exists(Path)) + { + Directory.Delete(Path, recursive: true); + } + } + catch + { + // ignored + } + } +} + +internal sealed class StubHttpMessageHandler : HttpMessageHandler +{ + private readonly Queue> _responses; + + public StubHttpMessageHandler(params Func[] handlers) + { + if (handlers is null || handlers.Length == 0) + { + throw new ArgumentException("At least one handler must be provided.", nameof(handlers)); + } + + _responses = new Queue>(handlers); + } + + protected override Task SendAsync(HttpRequestMessage request, CancellationToken cancellationToken) + { + var factory = _responses.Count > 1 ? _responses.Dequeue() : _responses.Peek(); + return Task.FromResult(factory(request, cancellationToken)); + } +} diff --git a/src/StellaOps.Cli.Tests/UnitTest1.cs b/src/StellaOps.Cli.Tests/UnitTest1.cs new file mode 100644 index 00000000..d8574043 --- /dev/null +++ b/src/StellaOps.Cli.Tests/UnitTest1.cs @@ -0,0 +1,10 @@ +namespace StellaOps.Cli.Tests; + +public class UnitTest1 +{ + [Fact] + public void Test1() + { + + } +} diff --git a/src/StellaOps.Cli.Tests/xunit.runner.json b/src/StellaOps.Cli.Tests/xunit.runner.json new file mode 100644 index 00000000..249d815c --- /dev/null +++ b/src/StellaOps.Cli.Tests/xunit.runner.json @@ -0,0 +1,3 @@ +{ + "$schema": "https://xunit.net/schema/current/xunit.runner.schema.json" +} diff --git a/src/StellaOps.Cli/AGENTS.md b/src/StellaOps.Cli/AGENTS.md new file mode 100644 index 00000000..77dc3e3a --- /dev/null +++ b/src/StellaOps.Cli/AGENTS.md @@ -0,0 +1,27 @@ +# StellaOps.Cli — Agent Brief + +## Mission +- Deliver an offline-capable command-line interface that drives StellaOps back-end operations: scanner distribution, scan execution, result uploads, and Feedser database lifecycle calls (init/resume/export). +- Honour StellaOps principles of determinism, observability, and offline-first behaviour while providing a polished operator experience. + +## Role Charter +| Role | Mandate | Collaboration | +| --- | --- | --- | +| **DevEx/CLI** | Own CLI UX, command routing, and configuration model. Ensure commands work with empty/default config and document overrides. | Coordinate with Backend/WebService for API contracts and with Docs for operator workflows. | +| **Ops Integrator** | Maintain integration paths for shell/dotnet/docker tooling. Validate that air-gapped runners can bootstrap required binaries. | Work with Feedser/Agent teams to mirror packaging and signing requirements. | +| **QA** | Provide command-level fixtures, golden outputs, and regression coverage (unit & smoke). Ensure commands respect cancellation and deterministic logging. | Partner with QA guild for shared harnesses and test data. | + +## Working Agreements +- Configuration is centralised in `StellaOps.Configuration`; always consume the bootstrapper instead of hand rolling builders. Env vars (`API_KEY`, `STELLAOPS_BACKEND_URL`, `StellaOps:*`) override JSON/YAML and default to empty values. +- Command verbs (`scanner`, `scan`, `db`, `config`) are wired through System.CommandLine 2.0; keep handlers composable, cancellation-aware, and unit-testable. +- `scanner download` must verify digests/signatures, install containers locally (docker load), and log artefact metadata. +- `scan run` must execute the container against a directory, materialise artefacts in `ResultsDirectory`, and auto-upload them on success; `scan upload` is the manual retry path. +- Emit structured console logs (single line, UTC timestamps) and honour offline-first expectations—no hidden network calls. +- Mirror repository guidance: stay within `src/StellaOps.Cli` unless collaborating via documented handshakes. +- Update `TASKS.md` as states change (TODO → DOING → DONE/BLOCKED) and record added tests/fixtures alongside implementation notes. + +## Reference Materials +- `docs/ARCHITECTURE_FEEDSER.md` for database operations surface area. +- Backend OpenAPI/contract docs (once available) for job triggers and scanner endpoints. +- Existing module AGENTS/TASKS files for style and coordination cues. +- `docs/09_API_CLI_REFERENCE.md` (section 3) for the user-facing synopsis of the CLI verbs and flags. diff --git a/src/StellaOps.Cli/Commands/CommandFactory.cs b/src/StellaOps.Cli/Commands/CommandFactory.cs new file mode 100644 index 00000000..efe3d956 --- /dev/null +++ b/src/StellaOps.Cli/Commands/CommandFactory.cs @@ -0,0 +1,324 @@ +using System; +using System.CommandLine; +using System.Threading; +using System.Threading.Tasks; +using StellaOps.Cli.Configuration; + +namespace StellaOps.Cli.Commands; + +internal static class CommandFactory +{ + public static RootCommand Create(IServiceProvider services, StellaOpsCliOptions options, CancellationToken cancellationToken) + { + var verboseOption = new Option("--verbose", new[] { "-v" }) + { + Description = "Enable verbose logging output." + }; + + var root = new RootCommand("StellaOps command-line interface") + { + TreatUnmatchedTokensAsErrors = true + }; + root.Add(verboseOption); + + root.Add(BuildScannerCommand(services, verboseOption, cancellationToken)); + root.Add(BuildScanCommand(services, options, verboseOption, cancellationToken)); + root.Add(BuildDatabaseCommand(services, verboseOption, cancellationToken)); + root.Add(BuildAuthCommand(services, options, verboseOption, cancellationToken)); + root.Add(BuildConfigCommand(options)); + + return root; + } + + private static Command BuildScannerCommand(IServiceProvider services, Option verboseOption, CancellationToken cancellationToken) + { + var scanner = new Command("scanner", "Manage scanner artifacts and lifecycle."); + + var download = new Command("download", "Download the latest scanner bundle."); + var channelOption = new Option("--channel", new[] { "-c" }) + { + Description = "Scanner channel (stable, beta, nightly)." + }; + + var outputOption = new Option("--output") + { + Description = "Optional output path for the downloaded bundle." + }; + + var overwriteOption = new Option("--overwrite") + { + Description = "Overwrite existing bundle if present." + }; + + var noInstallOption = new Option("--no-install") + { + Description = "Skip installing the scanner container after download." + }; + + download.Add(channelOption); + download.Add(outputOption); + download.Add(overwriteOption); + download.Add(noInstallOption); + + download.SetAction((parseResult, _) => + { + var channel = parseResult.GetValue(channelOption) ?? "stable"; + var output = parseResult.GetValue(outputOption); + var overwrite = parseResult.GetValue(overwriteOption); + var install = !parseResult.GetValue(noInstallOption); + var verbose = parseResult.GetValue(verboseOption); + + return CommandHandlers.HandleScannerDownloadAsync(services, channel, output, overwrite, install, verbose, cancellationToken); + }); + + scanner.Add(download); + return scanner; + } + + private static Command BuildScanCommand(IServiceProvider services, StellaOpsCliOptions options, Option verboseOption, CancellationToken cancellationToken) + { + var scan = new Command("scan", "Execute scanners and manage scan outputs."); + + var run = new Command("run", "Execute a scanner bundle with the configured runner."); + var runnerOption = new Option("--runner") + { + Description = "Execution runtime (dotnet, self, docker)." + }; + var entryOption = new Option("--entry") + { + Description = "Path to the scanner entrypoint or Docker image.", + Required = true + }; + var targetOption = new Option("--target") + { + Description = "Directory to scan.", + Required = true + }; + + var argsArgument = new Argument("scanner-args") + { + Arity = ArgumentArity.ZeroOrMore + }; + + run.Add(runnerOption); + run.Add(entryOption); + run.Add(targetOption); + run.Add(argsArgument); + + run.SetAction((parseResult, _) => + { + var runner = parseResult.GetValue(runnerOption) ?? options.DefaultRunner; + var entry = parseResult.GetValue(entryOption) ?? string.Empty; + var target = parseResult.GetValue(targetOption) ?? string.Empty; + var forwardedArgs = parseResult.GetValue(argsArgument) ?? Array.Empty(); + var verbose = parseResult.GetValue(verboseOption); + + return CommandHandlers.HandleScannerRunAsync(services, runner, entry, target, forwardedArgs, verbose, cancellationToken); + }); + + var upload = new Command("upload", "Upload completed scan results to the backend."); + var fileOption = new Option("--file") + { + Description = "Path to the scan result artifact.", + Required = true + }; + upload.Add(fileOption); + upload.SetAction((parseResult, _) => + { + var file = parseResult.GetValue(fileOption) ?? string.Empty; + var verbose = parseResult.GetValue(verboseOption); + return CommandHandlers.HandleScanUploadAsync(services, file, verbose, cancellationToken); + }); + + scan.Add(run); + scan.Add(upload); + return scan; + } + + private static Command BuildDatabaseCommand(IServiceProvider services, Option verboseOption, CancellationToken cancellationToken) + { + var db = new Command("db", "Trigger Feedser database operations via backend jobs."); + + var fetch = new Command("fetch", "Trigger connector fetch/parse/map stages."); + var sourceOption = new Option("--source") + { + Description = "Connector source identifier (e.g. redhat, osv, vmware).", + Required = true + }; + var stageOption = new Option("--stage") + { + Description = "Stage to trigger: fetch, parse, or map." + }; + var modeOption = new Option("--mode") + { + Description = "Optional connector-specific mode (init, resume, cursor)." + }; + + fetch.Add(sourceOption); + fetch.Add(stageOption); + fetch.Add(modeOption); + fetch.SetAction((parseResult, _) => + { + var source = parseResult.GetValue(sourceOption) ?? string.Empty; + var stage = parseResult.GetValue(stageOption) ?? "fetch"; + var mode = parseResult.GetValue(modeOption); + var verbose = parseResult.GetValue(verboseOption); + + return CommandHandlers.HandleConnectorJobAsync(services, source, stage, mode, verbose, cancellationToken); + }); + + var merge = new Command("merge", "Run canonical merge reconciliation."); + merge.SetAction((parseResult, _) => + { + var verbose = parseResult.GetValue(verboseOption); + return CommandHandlers.HandleMergeJobAsync(services, verbose, cancellationToken); + }); + + var export = new Command("export", "Run Feedser export jobs."); + var formatOption = new Option("--format") + { + Description = "Export format: json or trivy-db." + }; + var deltaOption = new Option("--delta") + { + Description = "Request a delta export when supported." + }; + var publishFullOption = new Option("--publish-full") + { + Description = "Override whether full exports push to ORAS (true/false)." + }; + var publishDeltaOption = new Option("--publish-delta") + { + Description = "Override whether delta exports push to ORAS (true/false)." + }; + var includeFullOption = new Option("--bundle-full") + { + Description = "Override whether offline bundles include full exports (true/false)." + }; + var includeDeltaOption = new Option("--bundle-delta") + { + Description = "Override whether offline bundles include delta exports (true/false)." + }; + + export.Add(formatOption); + export.Add(deltaOption); + export.Add(publishFullOption); + export.Add(publishDeltaOption); + export.Add(includeFullOption); + export.Add(includeDeltaOption); + export.SetAction((parseResult, _) => + { + var format = parseResult.GetValue(formatOption) ?? "json"; + var delta = parseResult.GetValue(deltaOption); + var publishFull = parseResult.GetValue(publishFullOption); + var publishDelta = parseResult.GetValue(publishDeltaOption); + var includeFull = parseResult.GetValue(includeFullOption); + var includeDelta = parseResult.GetValue(includeDeltaOption); + var verbose = parseResult.GetValue(verboseOption); + return CommandHandlers.HandleExportJobAsync(services, format, delta, publishFull, publishDelta, includeFull, includeDelta, verbose, cancellationToken); + }); + + db.Add(fetch); + db.Add(merge); + db.Add(export); + return db; + } + + private static Command BuildAuthCommand(IServiceProvider services, StellaOpsCliOptions options, Option verboseOption, CancellationToken cancellationToken) + { + var auth = new Command("auth", "Manage authentication with StellaOps Authority."); + + var login = new Command("login", "Acquire and cache access tokens using the configured credentials."); + var forceOption = new Option("--force") + { + Description = "Ignore existing cached tokens and force re-authentication." + }; + login.Add(forceOption); + login.SetAction((parseResult, _) => + { + var verbose = parseResult.GetValue(verboseOption); + var force = parseResult.GetValue(forceOption); + return CommandHandlers.HandleAuthLoginAsync(services, options, verbose, force, cancellationToken); + }); + + var logout = new Command("logout", "Remove cached tokens for the current credentials."); + logout.SetAction((parseResult, _) => + { + var verbose = parseResult.GetValue(verboseOption); + return CommandHandlers.HandleAuthLogoutAsync(services, options, verbose, cancellationToken); + }); + + var status = new Command("status", "Display cached token status."); + status.SetAction((parseResult, _) => + { + var verbose = parseResult.GetValue(verboseOption); + return CommandHandlers.HandleAuthStatusAsync(services, options, verbose, cancellationToken); + }); + + var whoami = new Command("whoami", "Display cached token claims (subject, scopes, expiry)."); + whoami.SetAction((parseResult, _) => + { + var verbose = parseResult.GetValue(verboseOption); + return CommandHandlers.HandleAuthWhoAmIAsync(services, options, verbose, cancellationToken); + }); + + auth.Add(login); + auth.Add(logout); + auth.Add(status); + auth.Add(whoami); + return auth; + } + + private static Command BuildConfigCommand(StellaOpsCliOptions options) + { + var config = new Command("config", "Inspect CLI configuration state."); + var show = new Command("show", "Display resolved configuration values."); + + show.SetAction((_, _) => + { + var authority = options.Authority ?? new StellaOpsCliAuthorityOptions(); + var lines = new[] + { + $"Backend URL: {MaskIfEmpty(options.BackendUrl)}", + $"API Key: {DescribeSecret(options.ApiKey)}", + $"Scanner Cache: {options.ScannerCacheDirectory}", + $"Results Directory: {options.ResultsDirectory}", + $"Default Runner: {options.DefaultRunner}", + $"Authority URL: {MaskIfEmpty(authority.Url)}", + $"Authority Client ID: {MaskIfEmpty(authority.ClientId)}", + $"Authority Client Secret: {DescribeSecret(authority.ClientSecret ?? string.Empty)}", + $"Authority Username: {MaskIfEmpty(authority.Username)}", + $"Authority Password: {DescribeSecret(authority.Password ?? string.Empty)}", + $"Authority Scope: {MaskIfEmpty(authority.Scope)}", + $"Authority Token Cache: {MaskIfEmpty(authority.TokenCacheDirectory ?? string.Empty)}" + }; + + foreach (var line in lines) + { + Console.WriteLine(line); + } + + return Task.CompletedTask; + }); + + config.Add(show); + return config; + } + + private static string MaskIfEmpty(string value) + => string.IsNullOrWhiteSpace(value) ? "" : value; + + private static string DescribeSecret(string value) + { + if (string.IsNullOrWhiteSpace(value)) + { + return ""; + } + + return value.Length switch + { + <= 4 => "****", + _ => $"{value[..2]}***{value[^2..]}" + }; + } +} diff --git a/src/StellaOps.Cli/Commands/CommandHandlers.cs b/src/StellaOps.Cli/Commands/CommandHandlers.cs new file mode 100644 index 00000000..1d72f754 --- /dev/null +++ b/src/StellaOps.Cli/Commands/CommandHandlers.cs @@ -0,0 +1,840 @@ +using System; +using System.Collections.Generic; +using System.Diagnostics; +using System.Globalization; +using System.IO; +using System.Text.Json; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Logging; +using Spectre.Console; +using StellaOps.Auth.Client; +using StellaOps.Cli.Configuration; +using StellaOps.Cli.Prompts; +using StellaOps.Cli.Services; +using StellaOps.Cli.Services.Models; +using StellaOps.Cli.Telemetry; + +namespace StellaOps.Cli.Commands; + +internal static class CommandHandlers +{ + public static async Task HandleScannerDownloadAsync( + IServiceProvider services, + string channel, + string? output, + bool overwrite, + bool install, + bool verbose, + CancellationToken cancellationToken) + { + await using var scope = services.CreateAsyncScope(); + var client = scope.ServiceProvider.GetRequiredService(); + var logger = scope.ServiceProvider.GetRequiredService().CreateLogger("scanner-download"); + var verbosity = scope.ServiceProvider.GetRequiredService(); + var previousLevel = verbosity.MinimumLevel; + verbosity.MinimumLevel = verbose ? LogLevel.Debug : LogLevel.Information; + using var activity = CliActivitySource.Instance.StartActivity("cli.scanner.download", ActivityKind.Client); + activity?.SetTag("stellaops.cli.command", "scanner download"); + activity?.SetTag("stellaops.cli.channel", channel); + using var duration = CliMetrics.MeasureCommandDuration("scanner download"); + + try + { + var result = await client.DownloadScannerAsync(channel, output ?? string.Empty, overwrite, verbose, cancellationToken).ConfigureAwait(false); + + if (result.FromCache) + { + logger.LogInformation("Using cached scanner at {Path}.", result.Path); + } + else + { + logger.LogInformation("Scanner downloaded to {Path} ({Size} bytes).", result.Path, result.SizeBytes); + } + + CliMetrics.RecordScannerDownload(channel, result.FromCache); + + if (install) + { + var installer = scope.ServiceProvider.GetRequiredService(); + await installer.InstallAsync(result.Path, verbose, cancellationToken).ConfigureAwait(false); + CliMetrics.RecordScannerInstall(channel); + } + + Environment.ExitCode = 0; + } + catch (Exception ex) + { + logger.LogError(ex, "Failed to download scanner bundle."); + Environment.ExitCode = 1; + } + finally + { + verbosity.MinimumLevel = previousLevel; + } + } + + public static async Task HandleScannerRunAsync( + IServiceProvider services, + string runner, + string entry, + string targetDirectory, + IReadOnlyList arguments, + bool verbose, + CancellationToken cancellationToken) + { + await using var scope = services.CreateAsyncScope(); + var executor = scope.ServiceProvider.GetRequiredService(); + var logger = scope.ServiceProvider.GetRequiredService().CreateLogger("scanner-run"); + var verbosity = scope.ServiceProvider.GetRequiredService(); + var previousLevel = verbosity.MinimumLevel; + verbosity.MinimumLevel = verbose ? LogLevel.Debug : LogLevel.Information; + using var activity = CliActivitySource.Instance.StartActivity("cli.scan.run", ActivityKind.Internal); + activity?.SetTag("stellaops.cli.command", "scan run"); + activity?.SetTag("stellaops.cli.runner", runner); + activity?.SetTag("stellaops.cli.entry", entry); + activity?.SetTag("stellaops.cli.target", targetDirectory); + using var duration = CliMetrics.MeasureCommandDuration("scan run"); + + try + { + var options = scope.ServiceProvider.GetRequiredService(); + var resultsDirectory = options.ResultsDirectory; + + var executionResult = await executor.RunAsync( + runner, + entry, + targetDirectory, + resultsDirectory, + arguments, + verbose, + cancellationToken).ConfigureAwait(false); + + Environment.ExitCode = executionResult.ExitCode; + CliMetrics.RecordScanRun(runner, executionResult.ExitCode); + + if (executionResult.ExitCode == 0) + { + var backend = scope.ServiceProvider.GetRequiredService(); + logger.LogInformation("Uploading scan artefact {Path}...", executionResult.ResultsPath); + await backend.UploadScanResultsAsync(executionResult.ResultsPath, cancellationToken).ConfigureAwait(false); + logger.LogInformation("Scan artefact uploaded."); + activity?.SetTag("stellaops.cli.results", executionResult.ResultsPath); + } + else + { + logger.LogWarning("Skipping automatic upload because scan exited with code {Code}.", executionResult.ExitCode); + } + + logger.LogInformation("Run metadata written to {Path}.", executionResult.RunMetadataPath); + activity?.SetTag("stellaops.cli.run_metadata", executionResult.RunMetadataPath); + } + catch (Exception ex) + { + logger.LogError(ex, "Scanner execution failed."); + Environment.ExitCode = 1; + } + finally + { + verbosity.MinimumLevel = previousLevel; + } + } + + public static async Task HandleScanUploadAsync( + IServiceProvider services, + string file, + bool verbose, + CancellationToken cancellationToken) + { + await using var scope = services.CreateAsyncScope(); + var client = scope.ServiceProvider.GetRequiredService(); + var logger = scope.ServiceProvider.GetRequiredService().CreateLogger("scanner-upload"); + var verbosity = scope.ServiceProvider.GetRequiredService(); + var previousLevel = verbosity.MinimumLevel; + verbosity.MinimumLevel = verbose ? LogLevel.Debug : LogLevel.Information; + using var activity = CliActivitySource.Instance.StartActivity("cli.scan.upload", ActivityKind.Client); + activity?.SetTag("stellaops.cli.command", "scan upload"); + activity?.SetTag("stellaops.cli.file", file); + using var duration = CliMetrics.MeasureCommandDuration("scan upload"); + + try + { + var path = Path.GetFullPath(file); + await client.UploadScanResultsAsync(path, cancellationToken).ConfigureAwait(false); + logger.LogInformation("Scan results uploaded successfully."); + Environment.ExitCode = 0; + } + catch (Exception ex) + { + logger.LogError(ex, "Failed to upload scan results."); + Environment.ExitCode = 1; + } + finally + { + verbosity.MinimumLevel = previousLevel; + } + } + + public static async Task HandleConnectorJobAsync( + IServiceProvider services, + string source, + string stage, + string? mode, + bool verbose, + CancellationToken cancellationToken) + { + await using var scope = services.CreateAsyncScope(); + var client = scope.ServiceProvider.GetRequiredService(); + var logger = scope.ServiceProvider.GetRequiredService().CreateLogger("db-connector"); + var verbosity = scope.ServiceProvider.GetRequiredService(); + var previousLevel = verbosity.MinimumLevel; + verbosity.MinimumLevel = verbose ? LogLevel.Debug : LogLevel.Information; + using var activity = CliActivitySource.Instance.StartActivity("cli.db.fetch", ActivityKind.Client); + activity?.SetTag("stellaops.cli.command", "db fetch"); + activity?.SetTag("stellaops.cli.source", source); + activity?.SetTag("stellaops.cli.stage", stage); + if (!string.IsNullOrWhiteSpace(mode)) + { + activity?.SetTag("stellaops.cli.mode", mode); + } + using var duration = CliMetrics.MeasureCommandDuration("db fetch"); + + try + { + var jobKind = $"source:{source}:{stage}"; + var parameters = new Dictionary(StringComparer.Ordinal); + if (!string.IsNullOrWhiteSpace(mode)) + { + parameters["mode"] = mode; + } + + await TriggerJobAsync(client, logger, jobKind, parameters, cancellationToken).ConfigureAwait(false); + } + catch (Exception ex) + { + logger.LogError(ex, "Connector job failed."); + Environment.ExitCode = 1; + } + finally + { + verbosity.MinimumLevel = previousLevel; + } + } + + public static async Task HandleMergeJobAsync( + IServiceProvider services, + bool verbose, + CancellationToken cancellationToken) + { + await using var scope = services.CreateAsyncScope(); + var client = scope.ServiceProvider.GetRequiredService(); + var logger = scope.ServiceProvider.GetRequiredService().CreateLogger("db-merge"); + var verbosity = scope.ServiceProvider.GetRequiredService(); + var previousLevel = verbosity.MinimumLevel; + verbosity.MinimumLevel = verbose ? LogLevel.Debug : LogLevel.Information; + using var activity = CliActivitySource.Instance.StartActivity("cli.db.merge", ActivityKind.Client); + activity?.SetTag("stellaops.cli.command", "db merge"); + using var duration = CliMetrics.MeasureCommandDuration("db merge"); + + try + { + await TriggerJobAsync(client, logger, "merge:reconcile", new Dictionary(StringComparer.Ordinal), cancellationToken).ConfigureAwait(false); + } + catch (Exception ex) + { + logger.LogError(ex, "Merge job failed."); + Environment.ExitCode = 1; + } + finally + { + verbosity.MinimumLevel = previousLevel; + } + } + + public static async Task HandleExportJobAsync( + IServiceProvider services, + string format, + bool delta, + bool? publishFull, + bool? publishDelta, + bool? includeFull, + bool? includeDelta, + bool verbose, + CancellationToken cancellationToken) + { + await using var scope = services.CreateAsyncScope(); + var client = scope.ServiceProvider.GetRequiredService(); + var logger = scope.ServiceProvider.GetRequiredService().CreateLogger("db-export"); + var verbosity = scope.ServiceProvider.GetRequiredService(); + var previousLevel = verbosity.MinimumLevel; + verbosity.MinimumLevel = verbose ? LogLevel.Debug : LogLevel.Information; + using var activity = CliActivitySource.Instance.StartActivity("cli.db.export", ActivityKind.Client); + activity?.SetTag("stellaops.cli.command", "db export"); + activity?.SetTag("stellaops.cli.format", format); + activity?.SetTag("stellaops.cli.delta", delta); + using var duration = CliMetrics.MeasureCommandDuration("db export"); + activity?.SetTag("stellaops.cli.publish_full", publishFull); + activity?.SetTag("stellaops.cli.publish_delta", publishDelta); + activity?.SetTag("stellaops.cli.include_full", includeFull); + activity?.SetTag("stellaops.cli.include_delta", includeDelta); + + try + { + var jobKind = format switch + { + "trivy-db" or "trivy" => "export:trivy-db", + _ => "export:json" + }; + + var isTrivy = jobKind == "export:trivy-db"; + if (isTrivy + && !publishFull.HasValue + && !publishDelta.HasValue + && !includeFull.HasValue + && !includeDelta.HasValue + && AnsiConsole.Profile.Capabilities.Interactive) + { + var overrides = TrivyDbExportPrompt.PromptOverrides(); + publishFull = overrides.publishFull; + publishDelta = overrides.publishDelta; + includeFull = overrides.includeFull; + includeDelta = overrides.includeDelta; + } + + var parameters = new Dictionary(StringComparer.Ordinal) + { + ["delta"] = delta + }; + if (publishFull.HasValue) + { + parameters["publishFull"] = publishFull.Value; + } + if (publishDelta.HasValue) + { + parameters["publishDelta"] = publishDelta.Value; + } + if (includeFull.HasValue) + { + parameters["includeFull"] = includeFull.Value; + } + if (includeDelta.HasValue) + { + parameters["includeDelta"] = includeDelta.Value; + } + + await TriggerJobAsync(client, logger, jobKind, parameters, cancellationToken).ConfigureAwait(false); + } + catch (Exception ex) + { + logger.LogError(ex, "Export job failed."); + Environment.ExitCode = 1; + } + finally + { + verbosity.MinimumLevel = previousLevel; + } + } + + public static async Task HandleAuthLoginAsync( + IServiceProvider services, + StellaOpsCliOptions options, + bool verbose, + bool force, + CancellationToken cancellationToken) + { + await using var scope = services.CreateAsyncScope(); + var logger = scope.ServiceProvider.GetRequiredService().CreateLogger("auth-login"); + Environment.ExitCode = 0; + + if (string.IsNullOrWhiteSpace(options.Authority?.Url)) + { + logger.LogError("Authority URL is not configured. Set STELLAOPS_AUTHORITY_URL or update your configuration."); + Environment.ExitCode = 1; + return; + } + + var tokenClient = scope.ServiceProvider.GetService(); + if (tokenClient is null) + { + logger.LogError("Authority client is not available. Ensure AddStellaOpsAuthClient is registered in Program.cs."); + Environment.ExitCode = 1; + return; + } + + var cacheKey = AuthorityTokenUtilities.BuildCacheKey(options); + if (string.IsNullOrWhiteSpace(cacheKey)) + { + logger.LogError("Authority configuration is incomplete; unable to determine cache key."); + Environment.ExitCode = 1; + return; + } + + try + { + if (force) + { + await tokenClient.ClearCachedTokenAsync(cacheKey, cancellationToken).ConfigureAwait(false); + } + + var scopeName = AuthorityTokenUtilities.ResolveScope(options); + StellaOpsTokenResult token; + + if (!string.IsNullOrWhiteSpace(options.Authority.Username)) + { + if (string.IsNullOrWhiteSpace(options.Authority.Password)) + { + logger.LogError("Authority password must be provided when username is configured."); + Environment.ExitCode = 1; + return; + } + + token = await tokenClient.RequestPasswordTokenAsync( + options.Authority.Username, + options.Authority.Password!, + scopeName, + cancellationToken).ConfigureAwait(false); + } + else + { + token = await tokenClient.RequestClientCredentialsTokenAsync(scopeName, cancellationToken).ConfigureAwait(false); + } + + await tokenClient.CacheTokenAsync(cacheKey, token.ToCacheEntry(), cancellationToken).ConfigureAwait(false); + + if (verbose) + { + logger.LogInformation("Authenticated with {Authority} (scopes: {Scopes}).", options.Authority.Url, string.Join(", ", token.Scopes)); + } + + logger.LogInformation("Login successful. Access token expires at {Expires}.", token.ExpiresAtUtc.ToString("u")); + } + catch (Exception ex) + { + logger.LogError(ex, "Authentication failed: {Message}", ex.Message); + Environment.ExitCode = 1; + } + } + + public static async Task HandleAuthLogoutAsync( + IServiceProvider services, + StellaOpsCliOptions options, + bool verbose, + CancellationToken cancellationToken) + { + await using var scope = services.CreateAsyncScope(); + var logger = scope.ServiceProvider.GetRequiredService().CreateLogger("auth-logout"); + Environment.ExitCode = 0; + + var tokenClient = scope.ServiceProvider.GetService(); + if (tokenClient is null) + { + logger.LogInformation("No authority client registered; nothing to remove."); + return; + } + + var cacheKey = AuthorityTokenUtilities.BuildCacheKey(options); + if (string.IsNullOrWhiteSpace(cacheKey)) + { + logger.LogInformation("Authority configuration missing; no cached tokens to remove."); + return; + } + + await tokenClient.ClearCachedTokenAsync(cacheKey, cancellationToken).ConfigureAwait(false); + if (verbose) + { + logger.LogInformation("Cleared cached token for {Authority}.", options.Authority?.Url ?? "authority"); + } + } + + public static async Task HandleAuthStatusAsync( + IServiceProvider services, + StellaOpsCliOptions options, + bool verbose, + CancellationToken cancellationToken) + { + await using var scope = services.CreateAsyncScope(); + var logger = scope.ServiceProvider.GetRequiredService().CreateLogger("auth-status"); + Environment.ExitCode = 0; + + if (string.IsNullOrWhiteSpace(options.Authority?.Url)) + { + logger.LogInformation("Authority URL not configured. Set STELLAOPS_AUTHORITY_URL and run 'auth login'."); + Environment.ExitCode = 1; + return; + } + + var tokenClient = scope.ServiceProvider.GetService(); + if (tokenClient is null) + { + logger.LogInformation("Authority client not registered; no cached tokens available."); + Environment.ExitCode = 1; + return; + } + + var cacheKey = AuthorityTokenUtilities.BuildCacheKey(options); + if (string.IsNullOrWhiteSpace(cacheKey)) + { + logger.LogInformation("Authority configuration incomplete; no cached tokens available."); + Environment.ExitCode = 1; + return; + } + + var entry = await tokenClient.GetCachedTokenAsync(cacheKey, cancellationToken).ConfigureAwait(false); + if (entry is null) + { + logger.LogInformation("No cached token for {Authority}. Run 'auth login' to authenticate.", options.Authority.Url); + Environment.ExitCode = 1; + return; + } + + logger.LogInformation("Cached token for {Authority} expires at {Expires}.", options.Authority.Url, entry.ExpiresAtUtc.ToString("u")); + if (verbose) + { + logger.LogInformation("Scopes: {Scopes}", string.Join(", ", entry.Scopes)); + } + } + + public static async Task HandleAuthWhoAmIAsync( + IServiceProvider services, + StellaOpsCliOptions options, + bool verbose, + CancellationToken cancellationToken) + { + await using var scope = services.CreateAsyncScope(); + var logger = scope.ServiceProvider.GetRequiredService().CreateLogger("auth-whoami"); + Environment.ExitCode = 0; + + if (string.IsNullOrWhiteSpace(options.Authority?.Url)) + { + logger.LogInformation("Authority URL not configured. Set STELLAOPS_AUTHORITY_URL and run 'auth login'."); + Environment.ExitCode = 1; + return; + } + + var tokenClient = scope.ServiceProvider.GetService(); + if (tokenClient is null) + { + logger.LogInformation("Authority client not registered; no cached tokens available."); + Environment.ExitCode = 1; + return; + } + + var cacheKey = AuthorityTokenUtilities.BuildCacheKey(options); + if (string.IsNullOrWhiteSpace(cacheKey)) + { + logger.LogInformation("Authority configuration incomplete; no cached tokens available."); + Environment.ExitCode = 1; + return; + } + + var entry = await tokenClient.GetCachedTokenAsync(cacheKey, cancellationToken).ConfigureAwait(false); + if (entry is null) + { + logger.LogInformation("No cached token for {Authority}. Run 'auth login' to authenticate.", options.Authority.Url); + Environment.ExitCode = 1; + return; + } + + var grantType = string.IsNullOrWhiteSpace(options.Authority.Username) ? "client_credentials" : "password"; + var now = DateTimeOffset.UtcNow; + var remaining = entry.ExpiresAtUtc - now; + if (remaining < TimeSpan.Zero) + { + remaining = TimeSpan.Zero; + } + + logger.LogInformation("Authority: {Authority}", options.Authority.Url); + logger.LogInformation("Grant type: {GrantType}", grantType); + logger.LogInformation("Token type: {TokenType}", entry.TokenType); + logger.LogInformation("Expires: {Expires} ({Remaining})", entry.ExpiresAtUtc.ToString("u"), FormatDuration(remaining)); + + if (entry.Scopes.Count > 0) + { + logger.LogInformation("Scopes: {Scopes}", string.Join(", ", entry.Scopes)); + } + + if (TryExtractJwtClaims(entry.AccessToken, out var claims, out var issuedAt, out var notBefore)) + { + if (claims.TryGetValue("sub", out var subject) && !string.IsNullOrWhiteSpace(subject)) + { + logger.LogInformation("Subject: {Subject}", subject); + } + + if (claims.TryGetValue("client_id", out var clientId) && !string.IsNullOrWhiteSpace(clientId)) + { + logger.LogInformation("Client ID (token): {ClientId}", clientId); + } + + if (claims.TryGetValue("aud", out var audience) && !string.IsNullOrWhiteSpace(audience)) + { + logger.LogInformation("Audience: {Audience}", audience); + } + + if (claims.TryGetValue("iss", out var issuer) && !string.IsNullOrWhiteSpace(issuer)) + { + logger.LogInformation("Issuer: {Issuer}", issuer); + } + + if (issuedAt is not null) + { + logger.LogInformation("Issued at: {IssuedAt}", issuedAt.Value.ToString("u")); + } + + if (notBefore is not null) + { + logger.LogInformation("Not before: {NotBefore}", notBefore.Value.ToString("u")); + } + + var extraClaims = CollectAdditionalClaims(claims); + if (extraClaims.Count > 0 && verbose) + { + logger.LogInformation("Additional claims: {Claims}", string.Join(", ", extraClaims)); + } + } + else + { + logger.LogInformation("Access token appears opaque; claims are unavailable."); + } + } + + private static string FormatDuration(TimeSpan duration) + { + if (duration <= TimeSpan.Zero) + { + return "expired"; + } + + if (duration.TotalDays >= 1) + { + var days = (int)duration.TotalDays; + var hours = duration.Hours; + return hours > 0 + ? FormattableString.Invariant($"{days}d {hours}h") + : FormattableString.Invariant($"{days}d"); + } + + if (duration.TotalHours >= 1) + { + return FormattableString.Invariant($"{(int)duration.TotalHours}h {duration.Minutes}m"); + } + + if (duration.TotalMinutes >= 1) + { + return FormattableString.Invariant($"{(int)duration.TotalMinutes}m {duration.Seconds}s"); + } + + return FormattableString.Invariant($"{duration.Seconds}s"); + } + + private static bool TryExtractJwtClaims( + string accessToken, + out Dictionary claims, + out DateTimeOffset? issuedAt, + out DateTimeOffset? notBefore) + { + claims = new Dictionary(StringComparer.OrdinalIgnoreCase); + issuedAt = null; + notBefore = null; + + if (string.IsNullOrWhiteSpace(accessToken)) + { + return false; + } + + var parts = accessToken.Split('.'); + if (parts.Length < 2) + { + return false; + } + + if (!TryDecodeBase64Url(parts[1], out var payloadBytes)) + { + return false; + } + + try + { + using var document = JsonDocument.Parse(payloadBytes); + foreach (var property in document.RootElement.EnumerateObject()) + { + var value = FormatJsonValue(property.Value); + claims[property.Name] = value; + + if (issuedAt is null && property.NameEquals("iat") && TryParseUnixSeconds(property.Value, out var parsedIat)) + { + issuedAt = parsedIat; + } + + if (notBefore is null && property.NameEquals("nbf") && TryParseUnixSeconds(property.Value, out var parsedNbf)) + { + notBefore = parsedNbf; + } + } + + return true; + } + catch (JsonException) + { + claims.Clear(); + issuedAt = null; + notBefore = null; + return false; + } + } + + private static bool TryDecodeBase64Url(string value, out byte[] bytes) + { + bytes = Array.Empty(); + + if (string.IsNullOrWhiteSpace(value)) + { + return false; + } + + var normalized = value.Replace('-', '+').Replace('_', '/'); + var padding = normalized.Length % 4; + if (padding is 2 or 3) + { + normalized = normalized.PadRight(normalized.Length + (4 - padding), '='); + } + else if (padding == 1) + { + return false; + } + + try + { + bytes = Convert.FromBase64String(normalized); + return true; + } + catch (FormatException) + { + return false; + } + } + + private static string FormatJsonValue(JsonElement element) + { + return element.ValueKind switch + { + JsonValueKind.String => element.GetString() ?? string.Empty, + JsonValueKind.Number => element.TryGetInt64(out var longValue) + ? longValue.ToString(CultureInfo.InvariantCulture) + : element.GetDouble().ToString(CultureInfo.InvariantCulture), + JsonValueKind.True => "true", + JsonValueKind.False => "false", + JsonValueKind.Null => "null", + JsonValueKind.Array => FormatArray(element), + JsonValueKind.Object => element.GetRawText(), + _ => element.GetRawText() + }; + } + + private static string FormatArray(JsonElement array) + { + var values = new List(); + foreach (var item in array.EnumerateArray()) + { + values.Add(FormatJsonValue(item)); + } + + return string.Join(", ", values); + } + + private static bool TryParseUnixSeconds(JsonElement element, out DateTimeOffset value) + { + value = default; + + if (element.ValueKind == JsonValueKind.Number) + { + if (element.TryGetInt64(out var seconds)) + { + value = DateTimeOffset.FromUnixTimeSeconds(seconds); + return true; + } + + if (element.TryGetDouble(out var doubleValue)) + { + value = DateTimeOffset.FromUnixTimeSeconds((long)doubleValue); + return true; + } + } + + if (element.ValueKind == JsonValueKind.String) + { + var text = element.GetString(); + if (!string.IsNullOrWhiteSpace(text) && long.TryParse(text, NumberStyles.Integer, CultureInfo.InvariantCulture, out var seconds)) + { + value = DateTimeOffset.FromUnixTimeSeconds(seconds); + return true; + } + } + + return false; + } + + private static List CollectAdditionalClaims(Dictionary claims) + { + var result = new List(); + foreach (var pair in claims) + { + if (CommonClaimNames.Contains(pair.Key)) + { + continue; + } + + result.Add(FormattableString.Invariant($"{pair.Key}={pair.Value}")); + } + + result.Sort(StringComparer.OrdinalIgnoreCase); + return result; + } + + private static readonly HashSet CommonClaimNames = new(StringComparer.OrdinalIgnoreCase) + { + "aud", + "client_id", + "exp", + "iat", + "iss", + "nbf", + "scope", + "scopes", + "sub", + "token_type", + "jti" + }; + + private static async Task TriggerJobAsync( + IBackendOperationsClient client, + ILogger logger, + string jobKind, + IDictionary parameters, + CancellationToken cancellationToken) + { + JobTriggerResult result = await client.TriggerJobAsync(jobKind, parameters, cancellationToken).ConfigureAwait(false); + if (result.Success) + { + if (!string.IsNullOrWhiteSpace(result.Location)) + { + logger.LogInformation("Job accepted. Track status at {Location}.", result.Location); + } + else if (result.Run is not null) + { + logger.LogInformation("Job accepted. RunId: {RunId} Status: {Status}", result.Run.RunId, result.Run.Status); + } + else + { + logger.LogInformation("Job accepted."); + } + + Environment.ExitCode = 0; + } + else + { + logger.LogError("Job '{JobKind}' failed: {Message}", jobKind, result.Message); + Environment.ExitCode = 1; + } + } +} diff --git a/src/StellaOps.Cli/Configuration/AuthorityTokenUtilities.cs b/src/StellaOps.Cli/Configuration/AuthorityTokenUtilities.cs new file mode 100644 index 00000000..8694a4c7 --- /dev/null +++ b/src/StellaOps.Cli/Configuration/AuthorityTokenUtilities.cs @@ -0,0 +1,34 @@ +using System; +using StellaOps.Auth.Abstractions; + +namespace StellaOps.Cli.Configuration; + +internal static class AuthorityTokenUtilities +{ + public static string ResolveScope(StellaOpsCliOptions options) + { + ArgumentNullException.ThrowIfNull(options); + + var scope = options.Authority?.Scope; + return string.IsNullOrWhiteSpace(scope) + ? StellaOpsScopes.FeedserJobsTrigger + : scope.Trim(); + } + + public static string BuildCacheKey(StellaOpsCliOptions options) + { + ArgumentNullException.ThrowIfNull(options); + + if (options.Authority is null) + { + return string.Empty; + } + + var scope = ResolveScope(options); + var credential = !string.IsNullOrWhiteSpace(options.Authority.Username) + ? $"user:{options.Authority.Username}" + : $"client:{options.Authority.ClientId}"; + + return $"{options.Authority.Url}|{credential}|{scope}"; + } +} diff --git a/src/StellaOps.Cli/Configuration/CliBootstrapper.cs b/src/StellaOps.Cli/Configuration/CliBootstrapper.cs new file mode 100644 index 00000000..a84f5d3c --- /dev/null +++ b/src/StellaOps.Cli/Configuration/CliBootstrapper.cs @@ -0,0 +1,278 @@ +using System; +using System.Collections.Generic; +using System.Globalization; +using System.IO; +using Microsoft.Extensions.Configuration; +using StellaOps.Configuration; +using StellaOps.Auth.Abstractions; + +namespace StellaOps.Cli.Configuration; + +public static class CliBootstrapper +{ + public static (StellaOpsCliOptions Options, IConfigurationRoot Configuration) Build(string[] args) + { + var bootstrap = StellaOpsConfigurationBootstrapper.Build(options => + { + options.BindingSection = "StellaOps"; + options.ConfigureBuilder = builder => + { + if (args.Length > 0) + { + builder.AddCommandLine(args); + } + }; + options.PostBind = (cliOptions, configuration) => + { + cliOptions.ApiKey = ResolveWithFallback(cliOptions.ApiKey, configuration, "API_KEY", "StellaOps:ApiKey", "ApiKey"); + cliOptions.BackendUrl = ResolveWithFallback(cliOptions.BackendUrl, configuration, "STELLAOPS_BACKEND_URL", "StellaOps:BackendUrl", "BackendUrl"); + cliOptions.ScannerSignaturePublicKeyPath = ResolveWithFallback(cliOptions.ScannerSignaturePublicKeyPath, configuration, "SCANNER_PUBLIC_KEY", "STELLAOPS_SCANNER_PUBLIC_KEY", "StellaOps:ScannerSignaturePublicKeyPath", "ScannerSignaturePublicKeyPath"); + + cliOptions.ApiKey = cliOptions.ApiKey?.Trim() ?? string.Empty; + cliOptions.BackendUrl = cliOptions.BackendUrl?.Trim() ?? string.Empty; + cliOptions.ScannerSignaturePublicKeyPath = cliOptions.ScannerSignaturePublicKeyPath?.Trim() ?? string.Empty; + + var attemptsRaw = ResolveWithFallback( + string.Empty, + configuration, + "SCANNER_DOWNLOAD_ATTEMPTS", + "STELLAOPS_SCANNER_DOWNLOAD_ATTEMPTS", + "StellaOps:ScannerDownloadAttempts", + "ScannerDownloadAttempts"); + + if (string.IsNullOrWhiteSpace(attemptsRaw)) + { + attemptsRaw = cliOptions.ScannerDownloadAttempts.ToString(CultureInfo.InvariantCulture); + } + + if (int.TryParse(attemptsRaw, NumberStyles.Integer, CultureInfo.InvariantCulture, out var parsedAttempts) && parsedAttempts > 0) + { + cliOptions.ScannerDownloadAttempts = parsedAttempts; + } + + if (cliOptions.ScannerDownloadAttempts <= 0) + { + cliOptions.ScannerDownloadAttempts = 3; + } + + cliOptions.Authority ??= new StellaOpsCliAuthorityOptions(); + var authority = cliOptions.Authority; + + authority.Url = ResolveWithFallback( + authority.Url, + configuration, + "STELLAOPS_AUTHORITY_URL", + "StellaOps:Authority:Url", + "Authority:Url", + "Authority:Issuer"); + + authority.ClientId = ResolveWithFallback( + authority.ClientId, + configuration, + "STELLAOPS_AUTHORITY_CLIENT_ID", + "StellaOps:Authority:ClientId", + "Authority:ClientId"); + + authority.ClientSecret = ResolveWithFallback( + authority.ClientSecret ?? string.Empty, + configuration, + "STELLAOPS_AUTHORITY_CLIENT_SECRET", + "StellaOps:Authority:ClientSecret", + "Authority:ClientSecret"); + + authority.Username = ResolveWithFallback( + authority.Username, + configuration, + "STELLAOPS_AUTHORITY_USERNAME", + "StellaOps:Authority:Username", + "Authority:Username"); + + authority.Password = ResolveWithFallback( + authority.Password ?? string.Empty, + configuration, + "STELLAOPS_AUTHORITY_PASSWORD", + "StellaOps:Authority:Password", + "Authority:Password"); + + authority.Scope = ResolveWithFallback( + authority.Scope, + configuration, + "STELLAOPS_AUTHORITY_SCOPE", + "StellaOps:Authority:Scope", + "Authority:Scope"); + + authority.TokenCacheDirectory = ResolveWithFallback( + authority.TokenCacheDirectory, + configuration, + "STELLAOPS_AUTHORITY_TOKEN_CACHE_DIR", + "StellaOps:Authority:TokenCacheDirectory", + "Authority:TokenCacheDirectory"); + + authority.Url = authority.Url?.Trim() ?? string.Empty; + authority.ClientId = authority.ClientId?.Trim() ?? string.Empty; + authority.ClientSecret = string.IsNullOrWhiteSpace(authority.ClientSecret) ? null : authority.ClientSecret.Trim(); + authority.Username = authority.Username?.Trim() ?? string.Empty; + authority.Password = string.IsNullOrWhiteSpace(authority.Password) ? null : authority.Password.Trim(); + authority.Scope = string.IsNullOrWhiteSpace(authority.Scope) ? StellaOpsScopes.FeedserJobsTrigger : authority.Scope.Trim(); + + authority.Resilience ??= new StellaOpsCliAuthorityResilienceOptions(); + authority.Resilience.RetryDelays ??= new List(); + var resilience = authority.Resilience; + + if (!resilience.EnableRetries.HasValue) + { + var raw = ResolveWithFallback( + string.Empty, + configuration, + "STELLAOPS_AUTHORITY_ENABLE_RETRIES", + "StellaOps:Authority:Resilience:EnableRetries", + "StellaOps:Authority:EnableRetries", + "Authority:Resilience:EnableRetries", + "Authority:EnableRetries"); + + if (TryParseBoolean(raw, out var parsed)) + { + resilience.EnableRetries = parsed; + } + } + + var retryDelaysRaw = ResolveWithFallback( + string.Empty, + configuration, + "STELLAOPS_AUTHORITY_RETRY_DELAYS", + "StellaOps:Authority:Resilience:RetryDelays", + "StellaOps:Authority:RetryDelays", + "Authority:Resilience:RetryDelays", + "Authority:RetryDelays"); + + if (!string.IsNullOrWhiteSpace(retryDelaysRaw)) + { + resilience.RetryDelays.Clear(); + foreach (var delay in ParseRetryDelays(retryDelaysRaw)) + { + if (delay > TimeSpan.Zero) + { + resilience.RetryDelays.Add(delay); + } + } + } + + if (!resilience.AllowOfflineCacheFallback.HasValue) + { + var raw = ResolveWithFallback( + string.Empty, + configuration, + "STELLAOPS_AUTHORITY_ALLOW_OFFLINE_CACHE_FALLBACK", + "StellaOps:Authority:Resilience:AllowOfflineCacheFallback", + "StellaOps:Authority:AllowOfflineCacheFallback", + "Authority:Resilience:AllowOfflineCacheFallback", + "Authority:AllowOfflineCacheFallback"); + + if (TryParseBoolean(raw, out var parsed)) + { + resilience.AllowOfflineCacheFallback = parsed; + } + } + + if (!resilience.OfflineCacheTolerance.HasValue) + { + var raw = ResolveWithFallback( + string.Empty, + configuration, + "STELLAOPS_AUTHORITY_OFFLINE_CACHE_TOLERANCE", + "StellaOps:Authority:Resilience:OfflineCacheTolerance", + "StellaOps:Authority:OfflineCacheTolerance", + "Authority:Resilience:OfflineCacheTolerance", + "Authority:OfflineCacheTolerance"); + + if (TimeSpan.TryParse(raw, CultureInfo.InvariantCulture, out var tolerance) && tolerance >= TimeSpan.Zero) + { + resilience.OfflineCacheTolerance = tolerance; + } + } + + var defaultTokenCache = GetDefaultTokenCacheDirectory(); + if (string.IsNullOrWhiteSpace(authority.TokenCacheDirectory)) + { + authority.TokenCacheDirectory = defaultTokenCache; + } + else + { + authority.TokenCacheDirectory = Path.GetFullPath(authority.TokenCacheDirectory); + } + }; + }); + + return (bootstrap.Options, bootstrap.Configuration); + } + + private static string ResolveWithFallback(string currentValue, IConfiguration configuration, params string[] keys) + { + if (!string.IsNullOrWhiteSpace(currentValue)) + { + return currentValue; + } + + foreach (var key in keys) + { + var value = configuration[key]; + if (!string.IsNullOrWhiteSpace(value)) + { + return value; + } + } + + return string.Empty; + } + + private static bool TryParseBoolean(string value, out bool parsed) + { + if (string.IsNullOrWhiteSpace(value)) + { + parsed = default; + return false; + } + + if (bool.TryParse(value, out parsed)) + { + return true; + } + + if (int.TryParse(value, NumberStyles.Integer, CultureInfo.InvariantCulture, out var numeric)) + { + parsed = numeric != 0; + return true; + } + + parsed = default; + return false; + } + + private static IEnumerable ParseRetryDelays(string raw) + { + if (string.IsNullOrWhiteSpace(raw)) + { + yield break; + } + + var separators = new[] { ',', ';', ' ' }; + foreach (var token in raw.Split(separators, StringSplitOptions.RemoveEmptyEntries | StringSplitOptions.TrimEntries)) + { + if (TimeSpan.TryParse(token, CultureInfo.InvariantCulture, out var delay) && delay > TimeSpan.Zero) + { + yield return delay; + } + } + } + + private static string GetDefaultTokenCacheDirectory() + { + var home = Environment.GetFolderPath(Environment.SpecialFolder.UserProfile); + if (string.IsNullOrWhiteSpace(home)) + { + home = AppContext.BaseDirectory; + } + + return Path.GetFullPath(Path.Combine(home, ".stellaops", "tokens")); + } +} diff --git a/src/StellaOps.Cli/Configuration/StellaOpsCliOptions.cs b/src/StellaOps.Cli/Configuration/StellaOpsCliOptions.cs new file mode 100644 index 00000000..0cacb7cb --- /dev/null +++ b/src/StellaOps.Cli/Configuration/StellaOpsCliOptions.cs @@ -0,0 +1,56 @@ +using System; +using System.Collections.Generic; +using StellaOps.Auth.Abstractions; + +namespace StellaOps.Cli.Configuration; + +public sealed class StellaOpsCliOptions +{ + public string ApiKey { get; set; } = string.Empty; + + public string BackendUrl { get; set; } = string.Empty; + + public string ScannerCacheDirectory { get; set; } = "scanners"; + + public string ResultsDirectory { get; set; } = "results"; + + public string DefaultRunner { get; set; } = "docker"; + + public string ScannerSignaturePublicKeyPath { get; set; } = string.Empty; + + public int ScannerDownloadAttempts { get; set; } = 3; + + public int ScanUploadAttempts { get; set; } = 3; + + public StellaOpsCliAuthorityOptions Authority { get; set; } = new(); +} + +public sealed class StellaOpsCliAuthorityOptions +{ + public string Url { get; set; } = string.Empty; + + public string ClientId { get; set; } = string.Empty; + + public string? ClientSecret { get; set; } + + public string Username { get; set; } = string.Empty; + + public string? Password { get; set; } + + public string Scope { get; set; } = StellaOpsScopes.FeedserJobsTrigger; + + public string TokenCacheDirectory { get; set; } = string.Empty; + + public StellaOpsCliAuthorityResilienceOptions Resilience { get; set; } = new(); +} + +public sealed class StellaOpsCliAuthorityResilienceOptions +{ + public bool? EnableRetries { get; set; } + + public IList RetryDelays { get; set; } = new List(); + + public bool? AllowOfflineCacheFallback { get; set; } + + public TimeSpan? OfflineCacheTolerance { get; set; } +} diff --git a/src/StellaOps.Cli/Program.cs b/src/StellaOps.Cli/Program.cs new file mode 100644 index 00000000..0d4b96d4 --- /dev/null +++ b/src/StellaOps.Cli/Program.cs @@ -0,0 +1,119 @@ +using System; +using System.CommandLine; +using System.IO; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Logging; +using StellaOps.Auth.Client; +using StellaOps.Cli.Commands; +using StellaOps.Cli.Configuration; +using StellaOps.Cli.Services; +using StellaOps.Cli.Telemetry; + +namespace StellaOps.Cli; + +internal static class Program +{ + internal static async Task Main(string[] args) + { + var (options, configuration) = CliBootstrapper.Build(args); + + var services = new ServiceCollection(); + services.AddSingleton(configuration); + services.AddSingleton(options); + + var verbosityState = new VerbosityState(); + services.AddSingleton(verbosityState); + + services.AddLogging(builder => + { + builder.ClearProviders(); + builder.AddSimpleConsole(logOptions => + { + logOptions.TimestampFormat = "HH:mm:ss "; + logOptions.SingleLine = true; + }); + builder.AddFilter((category, level) => level >= verbosityState.MinimumLevel); + }); + + if (!string.IsNullOrWhiteSpace(options.Authority.Url)) + { + services.AddStellaOpsAuthClient(clientOptions => + { + clientOptions.Authority = options.Authority.Url; + clientOptions.ClientId = options.Authority.ClientId ?? string.Empty; + clientOptions.ClientSecret = options.Authority.ClientSecret; + clientOptions.DefaultScopes.Clear(); + clientOptions.DefaultScopes.Add(string.IsNullOrWhiteSpace(options.Authority.Scope) + ? StellaOps.Auth.Abstractions.StellaOpsScopes.FeedserJobsTrigger + : options.Authority.Scope); + + var resilience = options.Authority.Resilience ?? new StellaOpsCliAuthorityResilienceOptions(); + clientOptions.EnableRetries = resilience.EnableRetries ?? true; + + if (resilience.RetryDelays is { Count: > 0 }) + { + clientOptions.RetryDelays.Clear(); + foreach (var delay in resilience.RetryDelays) + { + if (delay > TimeSpan.Zero) + { + clientOptions.RetryDelays.Add(delay); + } + } + } + + if (resilience.AllowOfflineCacheFallback.HasValue) + { + clientOptions.AllowOfflineCacheFallback = resilience.AllowOfflineCacheFallback.Value; + } + + if (resilience.OfflineCacheTolerance.HasValue && resilience.OfflineCacheTolerance.Value >= TimeSpan.Zero) + { + clientOptions.OfflineCacheTolerance = resilience.OfflineCacheTolerance.Value; + } + }); + + var cacheDirectory = options.Authority.TokenCacheDirectory; + if (!string.IsNullOrWhiteSpace(cacheDirectory)) + { + Directory.CreateDirectory(cacheDirectory); + services.AddStellaOpsFileTokenCache(cacheDirectory); + } + } + + services.AddHttpClient(client => + { + client.Timeout = TimeSpan.FromMinutes(5); + if (!string.IsNullOrWhiteSpace(options.BackendUrl) && + Uri.TryCreate(options.BackendUrl, UriKind.Absolute, out var backendUri)) + { + client.BaseAddress = backendUri; + } + }); + + services.AddSingleton(); + services.AddSingleton(); + + await using var serviceProvider = services.BuildServiceProvider(); + using var cts = new CancellationTokenSource(); + Console.CancelKeyPress += (_, eventArgs) => + { + eventArgs.Cancel = true; + cts.Cancel(); + }; + + var rootCommand = CommandFactory.Create(serviceProvider, options, cts.Token); + var commandConfiguration = new CommandLineConfiguration(rootCommand); + var commandExit = await commandConfiguration.InvokeAsync(args, cts.Token).ConfigureAwait(false); + + var finalExit = Environment.ExitCode != 0 ? Environment.ExitCode : commandExit; + if (cts.IsCancellationRequested && finalExit == 0) + { + finalExit = 130; // Typical POSIX cancellation exit code + } + + return finalExit; + } +} diff --git a/src/StellaOps.Cli/Prompts/TrivyDbExportPrompt.cs b/src/StellaOps.Cli/Prompts/TrivyDbExportPrompt.cs new file mode 100644 index 00000000..8754958f --- /dev/null +++ b/src/StellaOps.Cli/Prompts/TrivyDbExportPrompt.cs @@ -0,0 +1,52 @@ +using Spectre.Console; + +namespace StellaOps.Cli.Prompts; + +internal static class TrivyDbExportPrompt +{ + public static (bool? publishFull, bool? publishDelta, bool? includeFull, bool? includeDelta) PromptOverrides() + { + if (!AnsiConsole.Profile.Capabilities.Interactive) + { + return (null, null, null, null); + } + + AnsiConsole.Write( + new Panel("[bold]Trivy DB Export Overrides[/]") + .Border(BoxBorder.Rounded) + .Header("Trivy DB") + .Collapse()); + + var shouldOverride = AnsiConsole.Prompt( + new SelectionPrompt() + .Title("Adjust publishing or offline bundle behaviour?") + .AddChoices("Leave defaults", "Override")); + + if (shouldOverride == "Leave defaults") + { + return (null, null, null, null); + } + + var publishFull = PromptBoolean("Push full exports to ORAS?"); + var publishDelta = PromptBoolean("Push delta exports to ORAS?"); + var includeFull = PromptBoolean("Include full exports in offline bundle?"); + var includeDelta = PromptBoolean("Include delta exports in offline bundle?"); + + return (publishFull, publishDelta, includeFull, includeDelta); + } + + private static bool? PromptBoolean(string question) + { + var choice = AnsiConsole.Prompt( + new SelectionPrompt() + .Title($"{question} [grey](select override or keep default)[/]") + .AddChoices("Keep default", "Yes", "No")); + + return choice switch + { + "Yes" => true, + "No" => false, + _ => (bool?)null, + }; + } +} diff --git a/src/StellaOps.Cli/Properties/AssemblyInfo.cs b/src/StellaOps.Cli/Properties/AssemblyInfo.cs new file mode 100644 index 00000000..ea17f62c --- /dev/null +++ b/src/StellaOps.Cli/Properties/AssemblyInfo.cs @@ -0,0 +1,3 @@ +using System.Runtime.CompilerServices; + +[assembly: InternalsVisibleTo("StellaOps.Cli.Tests")] diff --git a/src/StellaOps.Cli/Services/BackendOperationsClient.cs b/src/StellaOps.Cli/Services/BackendOperationsClient.cs new file mode 100644 index 00000000..6e4df175 --- /dev/null +++ b/src/StellaOps.Cli/Services/BackendOperationsClient.cs @@ -0,0 +1,535 @@ +using System; +using System.Collections.Generic; +using System.IO; +using System.Net; +using System.Net.Http; +using System.Linq; +using System.Net.Http.Headers; +using System.Net.Http.Json; +using System.Globalization; +using System.Security.Cryptography; +using System.Text; +using System.Text.Json; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.Extensions.Logging; +using StellaOps.Auth.Abstractions; +using StellaOps.Auth.Client; +using StellaOps.Cli.Configuration; +using StellaOps.Cli.Services.Models; +using StellaOps.Cli.Services.Models.Transport; + +namespace StellaOps.Cli.Services; + +internal sealed class BackendOperationsClient : IBackendOperationsClient +{ + private static readonly JsonSerializerOptions SerializerOptions = new(JsonSerializerDefaults.Web); + private static readonly TimeSpan TokenRefreshSkew = TimeSpan.FromSeconds(30); + + private readonly HttpClient _httpClient; + private readonly StellaOpsCliOptions _options; + private readonly ILogger _logger; + private readonly IStellaOpsTokenClient? _tokenClient; + private readonly object _tokenSync = new(); + private string? _cachedAccessToken; + private DateTimeOffset _cachedAccessTokenExpiresAt = DateTimeOffset.MinValue; + + public BackendOperationsClient(HttpClient httpClient, StellaOpsCliOptions options, ILogger logger, IStellaOpsTokenClient? tokenClient = null) + { + _httpClient = httpClient ?? throw new ArgumentNullException(nameof(httpClient)); + _options = options ?? throw new ArgumentNullException(nameof(options)); + _logger = logger ?? throw new ArgumentNullException(nameof(logger)); + _tokenClient = tokenClient; + + if (!string.IsNullOrWhiteSpace(_options.BackendUrl) && httpClient.BaseAddress is null) + { + if (Uri.TryCreate(_options.BackendUrl, UriKind.Absolute, out var baseUri)) + { + httpClient.BaseAddress = baseUri; + } + } + } + + public async Task DownloadScannerAsync(string channel, string outputPath, bool overwrite, bool verbose, CancellationToken cancellationToken) + { + EnsureBackendConfigured(); + + channel = string.IsNullOrWhiteSpace(channel) ? "stable" : channel.Trim(); + outputPath = ResolveArtifactPath(outputPath, channel); + Directory.CreateDirectory(Path.GetDirectoryName(outputPath)!); + + if (!overwrite && File.Exists(outputPath)) + { + var existing = new FileInfo(outputPath); + _logger.LogInformation("Scanner artifact already cached at {Path} ({Size} bytes).", outputPath, existing.Length); + return new ScannerArtifactResult(outputPath, existing.Length, true); + } + + var attempt = 0; + var maxAttempts = Math.Max(1, _options.ScannerDownloadAttempts); + + while (true) + { + attempt++; + try + { + using var request = CreateRequest(HttpMethod.Get, $"api/scanner/artifacts/{channel}"); + await AuthorizeRequestAsync(request, cancellationToken).ConfigureAwait(false); + using var response = await _httpClient.SendAsync(request, HttpCompletionOption.ResponseHeadersRead, cancellationToken).ConfigureAwait(false); + if (!response.IsSuccessStatusCode) + { + var failure = await CreateFailureMessageAsync(response, cancellationToken).ConfigureAwait(false); + throw new InvalidOperationException(failure); + } + + return await ProcessScannerResponseAsync(response, outputPath, channel, verbose, cancellationToken).ConfigureAwait(false); + } + catch (Exception ex) when (attempt < maxAttempts) + { + var backoffSeconds = Math.Pow(2, attempt); + _logger.LogWarning(ex, "Scanner download attempt {Attempt}/{MaxAttempts} failed. Retrying in {Delay:F0}s...", attempt, maxAttempts, backoffSeconds); + await Task.Delay(TimeSpan.FromSeconds(backoffSeconds), cancellationToken).ConfigureAwait(false); + } + } + } + + private async Task ProcessScannerResponseAsync(HttpResponseMessage response, string outputPath, string channel, bool verbose, CancellationToken cancellationToken) + { + var tempFile = outputPath + ".tmp"; + await using (var payloadStream = await response.Content.ReadAsStreamAsync(cancellationToken).ConfigureAwait(false)) + await using (var fileStream = File.Create(tempFile)) + { + await payloadStream.CopyToAsync(fileStream, cancellationToken).ConfigureAwait(false); + } + + var expectedDigest = ExtractHeaderValue(response.Headers, "X-StellaOps-Digest"); + var signatureHeader = ExtractHeaderValue(response.Headers, "X-StellaOps-Signature"); + + var digestHex = await ValidateDigestAsync(tempFile, expectedDigest, cancellationToken).ConfigureAwait(false); + await ValidateSignatureAsync(signatureHeader, digestHex, verbose, cancellationToken).ConfigureAwait(false); + + if (verbose) + { + var signatureNote = string.IsNullOrWhiteSpace(signatureHeader) ? "no signature" : "signature validated"; + _logger.LogDebug("Scanner digest sha256:{Digest} ({SignatureNote}).", digestHex, signatureNote); + } + + if (File.Exists(outputPath)) + { + File.Delete(outputPath); + } + + File.Move(tempFile, outputPath); + + PersistMetadata(outputPath, channel, digestHex, signatureHeader, response); + + var downloaded = new FileInfo(outputPath); + _logger.LogInformation("Scanner downloaded to {Path} ({Size} bytes).", outputPath, downloaded.Length); + + return new ScannerArtifactResult(outputPath, downloaded.Length, false); + } + + public async Task UploadScanResultsAsync(string filePath, CancellationToken cancellationToken) + { + EnsureBackendConfigured(); + + if (!File.Exists(filePath)) + { + throw new FileNotFoundException("Scan result file not found.", filePath); + } + + var maxAttempts = Math.Max(1, _options.ScanUploadAttempts); + var attempt = 0; + + while (true) + { + attempt++; + try + { + using var content = new MultipartFormDataContent(); + await using var fileStream = File.OpenRead(filePath); + var streamContent = new StreamContent(fileStream); + streamContent.Headers.ContentType = new MediaTypeHeaderValue("application/octet-stream"); + content.Add(streamContent, "file", Path.GetFileName(filePath)); + + using var request = CreateRequest(HttpMethod.Post, "api/scanner/results"); + await AuthorizeRequestAsync(request, cancellationToken).ConfigureAwait(false); + request.Content = content; + + using var response = await _httpClient.SendAsync(request, cancellationToken).ConfigureAwait(false); + if (response.IsSuccessStatusCode) + { + _logger.LogInformation("Scan results uploaded from {Path}.", filePath); + return; + } + + var failure = await CreateFailureMessageAsync(response, cancellationToken).ConfigureAwait(false); + if (attempt >= maxAttempts) + { + throw new InvalidOperationException(failure); + } + + var delay = GetRetryDelay(response, attempt); + _logger.LogWarning( + "Scan upload attempt {Attempt}/{MaxAttempts} failed ({Reason}). Retrying in {Delay:F1}s...", + attempt, + maxAttempts, + failure, + delay.TotalSeconds); + await Task.Delay(delay, cancellationToken).ConfigureAwait(false); + } + catch (Exception ex) when (attempt < maxAttempts) + { + var delay = TimeSpan.FromSeconds(Math.Pow(2, attempt)); + _logger.LogWarning( + ex, + "Scan upload attempt {Attempt}/{MaxAttempts} threw an exception. Retrying in {Delay:F1}s...", + attempt, + maxAttempts, + delay.TotalSeconds); + await Task.Delay(delay, cancellationToken).ConfigureAwait(false); + } + } + } + + public async Task TriggerJobAsync(string jobKind, IDictionary parameters, CancellationToken cancellationToken) + { + EnsureBackendConfigured(); + + if (string.IsNullOrWhiteSpace(jobKind)) + { + throw new ArgumentException("Job kind must be provided.", nameof(jobKind)); + } + + var requestBody = new JobTriggerRequest + { + Trigger = "cli", + Parameters = parameters is null ? new Dictionary(StringComparer.Ordinal) : new Dictionary(parameters, StringComparer.Ordinal) + }; + + var request = CreateRequest(HttpMethod.Post, $"jobs/{jobKind}"); + await AuthorizeRequestAsync(request, cancellationToken).ConfigureAwait(false); + request.Content = JsonContent.Create(requestBody, options: SerializerOptions); + + using var response = await _httpClient.SendAsync(request, cancellationToken).ConfigureAwait(false); + if (response.StatusCode == HttpStatusCode.Accepted) + { + JobRunResponse? run = null; + if (response.Content.Headers.ContentLength is > 0) + { + try + { + run = await response.Content.ReadFromJsonAsync(SerializerOptions, cancellationToken).ConfigureAwait(false); + } + catch (JsonException ex) + { + _logger.LogWarning(ex, "Failed to deserialize job run response for job kind {Kind}.", jobKind); + } + } + + var location = response.Headers.Location?.ToString(); + return new JobTriggerResult(true, "Accepted", location, run); + } + + var failureMessage = await CreateFailureMessageAsync(response, cancellationToken).ConfigureAwait(false); + return new JobTriggerResult(false, failureMessage, null, null); + } + + private HttpRequestMessage CreateRequest(HttpMethod method, string relativeUri) + { + if (!Uri.TryCreate(relativeUri, UriKind.RelativeOrAbsolute, out var requestUri)) + { + throw new InvalidOperationException($"Invalid request URI '{relativeUri}'."); + } + + if (requestUri.IsAbsoluteUri) + { + // Nothing to normalize. + } + else + { + requestUri = new Uri(relativeUri.TrimStart('/'), UriKind.Relative); + } + + return new HttpRequestMessage(method, requestUri); + } + + private async Task AuthorizeRequestAsync(HttpRequestMessage request, CancellationToken cancellationToken) + { + var token = await ResolveAccessTokenAsync(cancellationToken).ConfigureAwait(false); + if (!string.IsNullOrWhiteSpace(token)) + { + request.Headers.Authorization = new AuthenticationHeaderValue("Bearer", token); + } + } + + private async Task ResolveAccessTokenAsync(CancellationToken cancellationToken) + { + if (!string.IsNullOrWhiteSpace(_options.ApiKey)) + { + return _options.ApiKey; + } + + if (_tokenClient is null || string.IsNullOrWhiteSpace(_options.Authority.Url)) + { + return null; + } + + var now = DateTimeOffset.UtcNow; + + lock (_tokenSync) + { + if (!string.IsNullOrEmpty(_cachedAccessToken) && now < _cachedAccessTokenExpiresAt - TokenRefreshSkew) + { + return _cachedAccessToken; + } + } + + var cacheKey = AuthorityTokenUtilities.BuildCacheKey(_options); + var cachedEntry = await _tokenClient.GetCachedTokenAsync(cacheKey, cancellationToken).ConfigureAwait(false); + if (cachedEntry is not null && now < cachedEntry.ExpiresAtUtc - TokenRefreshSkew) + { + lock (_tokenSync) + { + _cachedAccessToken = cachedEntry.AccessToken; + _cachedAccessTokenExpiresAt = cachedEntry.ExpiresAtUtc; + return _cachedAccessToken; + } + } + + var scope = AuthorityTokenUtilities.ResolveScope(_options); + + StellaOpsTokenResult token; + if (!string.IsNullOrWhiteSpace(_options.Authority.Username)) + { + if (string.IsNullOrWhiteSpace(_options.Authority.Password)) + { + throw new InvalidOperationException("Authority password must be configured when username is provided."); + } + + token = await _tokenClient.RequestPasswordTokenAsync( + _options.Authority.Username, + _options.Authority.Password!, + scope, + cancellationToken).ConfigureAwait(false); + } + else + { + token = await _tokenClient.RequestClientCredentialsTokenAsync(scope, cancellationToken).ConfigureAwait(false); + } + + await _tokenClient.CacheTokenAsync(cacheKey, token.ToCacheEntry(), cancellationToken).ConfigureAwait(false); + + lock (_tokenSync) + { + _cachedAccessToken = token.AccessToken; + _cachedAccessTokenExpiresAt = token.ExpiresAtUtc; + return _cachedAccessToken; + } + } + + private void EnsureBackendConfigured() + { + if (_httpClient.BaseAddress is null) + { + throw new InvalidOperationException("Backend URL is not configured. Provide STELLAOPS_BACKEND_URL or configure appsettings."); + } + } + + private string ResolveArtifactPath(string outputPath, string channel) + { + if (!string.IsNullOrWhiteSpace(outputPath)) + { + return Path.GetFullPath(outputPath); + } + + var directory = string.IsNullOrWhiteSpace(_options.ScannerCacheDirectory) + ? Directory.GetCurrentDirectory() + : Path.GetFullPath(_options.ScannerCacheDirectory); + + Directory.CreateDirectory(directory); + var fileName = $"stellaops-scanner-{channel}.tar.gz"; + return Path.Combine(directory, fileName); + } + + private async Task CreateFailureMessageAsync(HttpResponseMessage response, CancellationToken cancellationToken) + { + var statusCode = (int)response.StatusCode; + var builder = new StringBuilder(); + builder.Append("Backend request failed with status "); + builder.Append(statusCode); + builder.Append(' '); + builder.Append(response.ReasonPhrase ?? "Unknown"); + + if (response.Content.Headers.ContentLength is > 0) + { + try + { + var problem = await response.Content.ReadFromJsonAsync(SerializerOptions, cancellationToken).ConfigureAwait(false); + if (problem is not null) + { + if (!string.IsNullOrWhiteSpace(problem.Title)) + { + builder.AppendLine().Append(problem.Title); + } + + if (!string.IsNullOrWhiteSpace(problem.Detail)) + { + builder.AppendLine().Append(problem.Detail); + } + } + } + catch (JsonException) + { + var raw = await response.Content.ReadAsStringAsync(cancellationToken).ConfigureAwait(false); + if (!string.IsNullOrWhiteSpace(raw)) + { + builder.AppendLine().Append(raw); + } + } + } + + return builder.ToString(); + } + + private static string? ExtractHeaderValue(HttpResponseHeaders headers, string name) + { + if (headers.TryGetValues(name, out var values)) + { + return values.FirstOrDefault(); + } + + return null; + } + + private async Task ValidateDigestAsync(string filePath, string? expectedDigest, CancellationToken cancellationToken) + { + string digestHex; + await using (var stream = File.OpenRead(filePath)) + { + var hash = await SHA256.HashDataAsync(stream, cancellationToken).ConfigureAwait(false); + digestHex = Convert.ToHexString(hash).ToLowerInvariant(); + } + + if (!string.IsNullOrWhiteSpace(expectedDigest)) + { + var normalized = NormalizeDigest(expectedDigest); + if (!normalized.Equals(digestHex, StringComparison.OrdinalIgnoreCase)) + { + File.Delete(filePath); + throw new InvalidOperationException($"Scanner digest mismatch. Expected sha256:{normalized}, calculated sha256:{digestHex}."); + } + } + else + { + _logger.LogWarning("Scanner download missing X-StellaOps-Digest header; relying on computed digest only."); + } + + return digestHex; + } + + private static string NormalizeDigest(string digest) + { + if (digest.StartsWith("sha256:", StringComparison.OrdinalIgnoreCase)) + { + return digest[7..]; + } + + return digest; + } + + private async Task ValidateSignatureAsync(string? signatureHeader, string digestHex, bool verbose, CancellationToken cancellationToken) + { + if (string.IsNullOrWhiteSpace(_options.ScannerSignaturePublicKeyPath)) + { + if (!string.IsNullOrWhiteSpace(signatureHeader)) + { + _logger.LogDebug("Signature header present but no public key configured; skipping validation."); + } + return; + } + + if (string.IsNullOrWhiteSpace(signatureHeader)) + { + throw new InvalidOperationException("Scanner signature missing while a public key is configured."); + } + + var publicKeyPath = Path.GetFullPath(_options.ScannerSignaturePublicKeyPath); + if (!File.Exists(publicKeyPath)) + { + throw new FileNotFoundException("Scanner signature public key not found.", publicKeyPath); + } + + var signatureBytes = Convert.FromBase64String(signatureHeader); + var digestBytes = Convert.FromHexString(digestHex); + + var pem = await File.ReadAllTextAsync(publicKeyPath, cancellationToken).ConfigureAwait(false); + using var rsa = RSA.Create(); + rsa.ImportFromPem(pem); + + var valid = rsa.VerifyHash(digestBytes, signatureBytes, HashAlgorithmName.SHA256, RSASignaturePadding.Pkcs1); + if (!valid) + { + throw new InvalidOperationException("Scanner signature validation failed."); + } + + if (verbose) + { + _logger.LogDebug("Scanner signature validated using key {KeyPath}.", publicKeyPath); + } + } + + private void PersistMetadata(string outputPath, string channel, string digestHex, string? signatureHeader, HttpResponseMessage response) + { + var metadata = new + { + channel, + digest = $"sha256:{digestHex}", + signature = signatureHeader, + downloadedAt = DateTimeOffset.UtcNow, + source = response.RequestMessage?.RequestUri?.ToString(), + sizeBytes = new FileInfo(outputPath).Length, + headers = new + { + etag = response.Headers.ETag?.Tag, + lastModified = response.Content.Headers.LastModified, + contentType = response.Content.Headers.ContentType?.ToString() + } + }; + + var metadataPath = outputPath + ".metadata.json"; + var json = JsonSerializer.Serialize(metadata, new JsonSerializerOptions + { + WriteIndented = true + }); + + File.WriteAllText(metadataPath, json); + } + + private static TimeSpan GetRetryDelay(HttpResponseMessage response, int attempt) + { + if (response.Headers.TryGetValues("Retry-After", out var retryValues)) + { + var value = retryValues.FirstOrDefault(); + if (!string.IsNullOrWhiteSpace(value)) + { + if (int.TryParse(value, NumberStyles.Integer, CultureInfo.InvariantCulture, out var seconds) && seconds >= 0) + { + return TimeSpan.FromSeconds(Math.Min(seconds, 300)); + } + + if (DateTimeOffset.TryParse(value, CultureInfo.InvariantCulture, DateTimeStyles.AssumeUniversal | DateTimeStyles.AdjustToUniversal, out var when)) + { + var delta = when - DateTimeOffset.UtcNow; + if (delta > TimeSpan.Zero) + { + return delta < TimeSpan.FromMinutes(5) ? delta : TimeSpan.FromMinutes(5); + } + } + } + } + + var fallbackSeconds = Math.Min(60, Math.Pow(2, attempt)); + return TimeSpan.FromSeconds(fallbackSeconds); + } +} diff --git a/src/StellaOps.Cli/Services/IBackendOperationsClient.cs b/src/StellaOps.Cli/Services/IBackendOperationsClient.cs new file mode 100644 index 00000000..b593524b --- /dev/null +++ b/src/StellaOps.Cli/Services/IBackendOperationsClient.cs @@ -0,0 +1,16 @@ +using System.Collections.Generic; +using System.Threading; +using System.Threading.Tasks; +using StellaOps.Cli.Configuration; +using StellaOps.Cli.Services.Models; + +namespace StellaOps.Cli.Services; + +internal interface IBackendOperationsClient +{ + Task DownloadScannerAsync(string channel, string outputPath, bool overwrite, bool verbose, CancellationToken cancellationToken); + + Task UploadScanResultsAsync(string filePath, CancellationToken cancellationToken); + + Task TriggerJobAsync(string jobKind, IDictionary parameters, CancellationToken cancellationToken); +} diff --git a/src/StellaOps.Cli/Services/IScannerExecutor.cs b/src/StellaOps.Cli/Services/IScannerExecutor.cs new file mode 100644 index 00000000..11069408 --- /dev/null +++ b/src/StellaOps.Cli/Services/IScannerExecutor.cs @@ -0,0 +1,17 @@ +using System.Collections.Generic; +using System.Threading; +using System.Threading.Tasks; + +namespace StellaOps.Cli.Services; + +internal interface IScannerExecutor +{ + Task RunAsync( + string runner, + string entry, + string targetDirectory, + string resultsDirectory, + IReadOnlyList arguments, + bool verbose, + CancellationToken cancellationToken); +} diff --git a/src/StellaOps.Cli/Services/IScannerInstaller.cs b/src/StellaOps.Cli/Services/IScannerInstaller.cs new file mode 100644 index 00000000..9d2013e4 --- /dev/null +++ b/src/StellaOps.Cli/Services/IScannerInstaller.cs @@ -0,0 +1,9 @@ +using System.Threading; +using System.Threading.Tasks; + +namespace StellaOps.Cli.Services; + +internal interface IScannerInstaller +{ + Task InstallAsync(string artifactPath, bool verbose, CancellationToken cancellationToken); +} diff --git a/src/StellaOps.Cli/Services/Models/JobTriggerResult.cs b/src/StellaOps.Cli/Services/Models/JobTriggerResult.cs new file mode 100644 index 00000000..e145901f --- /dev/null +++ b/src/StellaOps.Cli/Services/Models/JobTriggerResult.cs @@ -0,0 +1,9 @@ +using StellaOps.Cli.Services.Models.Transport; + +namespace StellaOps.Cli.Services.Models; + +internal sealed record JobTriggerResult( + bool Success, + string Message, + string? Location, + JobRunResponse? Run); diff --git a/src/StellaOps.Cli/Services/Models/ScannerArtifactResult.cs b/src/StellaOps.Cli/Services/Models/ScannerArtifactResult.cs new file mode 100644 index 00000000..e72b938a --- /dev/null +++ b/src/StellaOps.Cli/Services/Models/ScannerArtifactResult.cs @@ -0,0 +1,3 @@ +namespace StellaOps.Cli.Services.Models; + +internal sealed record ScannerArtifactResult(string Path, long SizeBytes, bool FromCache); diff --git a/src/StellaOps.Cli/Services/Models/Transport/JobRunResponse.cs b/src/StellaOps.Cli/Services/Models/Transport/JobRunResponse.cs new file mode 100644 index 00000000..2c36b52e --- /dev/null +++ b/src/StellaOps.Cli/Services/Models/Transport/JobRunResponse.cs @@ -0,0 +1,27 @@ +using System; +using System.Collections.Generic; + +namespace StellaOps.Cli.Services.Models.Transport; + +internal sealed class JobRunResponse +{ + public Guid RunId { get; set; } + + public string Kind { get; set; } = string.Empty; + + public string Status { get; set; } = string.Empty; + + public string Trigger { get; set; } = string.Empty; + + public DateTimeOffset CreatedAt { get; set; } + + public DateTimeOffset? StartedAt { get; set; } + + public DateTimeOffset? CompletedAt { get; set; } + + public string? Error { get; set; } + + public TimeSpan? Duration { get; set; } + + public IReadOnlyDictionary Parameters { get; set; } = new Dictionary(StringComparer.Ordinal); +} diff --git a/src/StellaOps.Cli/Services/Models/Transport/JobTriggerRequest.cs b/src/StellaOps.Cli/Services/Models/Transport/JobTriggerRequest.cs new file mode 100644 index 00000000..d2c5a5d8 --- /dev/null +++ b/src/StellaOps.Cli/Services/Models/Transport/JobTriggerRequest.cs @@ -0,0 +1,10 @@ +using System.Collections.Generic; + +namespace StellaOps.Cli.Services.Models.Transport; + +internal sealed class JobTriggerRequest +{ + public string Trigger { get; set; } = "cli"; + + public Dictionary Parameters { get; set; } = new(StringComparer.Ordinal); +} diff --git a/src/StellaOps.Cli/Services/Models/Transport/ProblemDocument.cs b/src/StellaOps.Cli/Services/Models/Transport/ProblemDocument.cs new file mode 100644 index 00000000..468c0534 --- /dev/null +++ b/src/StellaOps.Cli/Services/Models/Transport/ProblemDocument.cs @@ -0,0 +1,18 @@ +using System.Collections.Generic; + +namespace StellaOps.Cli.Services.Models.Transport; + +internal sealed class ProblemDocument +{ + public string? Type { get; set; } + + public string? Title { get; set; } + + public string? Detail { get; set; } + + public int? Status { get; set; } + + public string? Instance { get; set; } + + public Dictionary? Extensions { get; set; } +} diff --git a/src/StellaOps.Cli/Services/ScannerExecutionResult.cs b/src/StellaOps.Cli/Services/ScannerExecutionResult.cs new file mode 100644 index 00000000..80458807 --- /dev/null +++ b/src/StellaOps.Cli/Services/ScannerExecutionResult.cs @@ -0,0 +1,3 @@ +namespace StellaOps.Cli.Services; + +internal sealed record ScannerExecutionResult(int ExitCode, string ResultsPath, string RunMetadataPath); diff --git a/src/StellaOps.Cli/Services/ScannerExecutor.cs b/src/StellaOps.Cli/Services/ScannerExecutor.cs new file mode 100644 index 00000000..ca1724cc --- /dev/null +++ b/src/StellaOps.Cli/Services/ScannerExecutor.cs @@ -0,0 +1,329 @@ +using System; +using System.Collections.Generic; +using System.Diagnostics; +using System.IO; +using System.Linq; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.Extensions.Logging; +using System.Text.Json; + +namespace StellaOps.Cli.Services; + +internal sealed class ScannerExecutor : IScannerExecutor +{ + private readonly ILogger _logger; + + public ScannerExecutor(ILogger logger) + { + _logger = logger ?? throw new ArgumentNullException(nameof(logger)); + } + + public async Task RunAsync( + string runner, + string entry, + string targetDirectory, + string resultsDirectory, + IReadOnlyList arguments, + bool verbose, + CancellationToken cancellationToken) + { + if (string.IsNullOrWhiteSpace(targetDirectory)) + { + throw new ArgumentException("Target directory must be provided.", nameof(targetDirectory)); + } + + runner = string.IsNullOrWhiteSpace(runner) ? "docker" : runner.Trim().ToLowerInvariant(); + entry = entry?.Trim() ?? string.Empty; + + var normalizedTarget = Path.GetFullPath(targetDirectory); + if (!Directory.Exists(normalizedTarget)) + { + throw new DirectoryNotFoundException($"Scan target directory '{normalizedTarget}' does not exist."); + } + + resultsDirectory = string.IsNullOrWhiteSpace(resultsDirectory) + ? Path.Combine(Directory.GetCurrentDirectory(), "scan-results") + : Path.GetFullPath(resultsDirectory); + + Directory.CreateDirectory(resultsDirectory); + var executionTimestamp = DateTimeOffset.UtcNow; + var baselineFiles = Directory.GetFiles(resultsDirectory, "*", SearchOption.AllDirectories); + var baseline = new HashSet(baselineFiles, StringComparer.OrdinalIgnoreCase); + + var startInfo = BuildProcessStartInfo(runner, entry, normalizedTarget, resultsDirectory, arguments); + using var process = new Process { StartInfo = startInfo, EnableRaisingEvents = true }; + + var stdout = new List(); + var stderr = new List(); + + process.OutputDataReceived += (_, args) => + { + if (args.Data is null) + { + return; + } + + stdout.Add(args.Data); + if (verbose) + { + _logger.LogInformation("[scan] {Line}", args.Data); + } + }; + + process.ErrorDataReceived += (_, args) => + { + if (args.Data is null) + { + return; + } + + stderr.Add(args.Data); + _logger.LogError("[scan] {Line}", args.Data); + }; + + _logger.LogInformation("Launching scanner via {Runner} (entry: {Entry})...", runner, entry); + if (!process.Start()) + { + throw new InvalidOperationException("Failed to start scanner process."); + } + + process.BeginOutputReadLine(); + process.BeginErrorReadLine(); + + await process.WaitForExitAsync(cancellationToken).ConfigureAwait(false); + var completionTimestamp = DateTimeOffset.UtcNow; + + if (process.ExitCode == 0) + { + _logger.LogInformation("Scanner completed successfully."); + } + else + { + _logger.LogWarning("Scanner exited with code {Code}.", process.ExitCode); + } + + var resultsPath = ResolveResultsPath(resultsDirectory, executionTimestamp, baseline); + if (string.IsNullOrWhiteSpace(resultsPath)) + { + resultsPath = CreatePlaceholderResult(resultsDirectory); + } + + var metadataPath = WriteRunMetadata( + resultsDirectory, + executionTimestamp, + completionTimestamp, + runner, + entry, + normalizedTarget, + resultsPath, + arguments, + process.ExitCode, + stdout, + stderr); + + return new ScannerExecutionResult(process.ExitCode, resultsPath, metadataPath); + } + + private ProcessStartInfo BuildProcessStartInfo( + string runner, + string entry, + string targetDirectory, + string resultsDirectory, + IReadOnlyList args) + { + return runner switch + { + "self" or "native" => BuildNativeStartInfo(entry, args), + "dotnet" => BuildDotNetStartInfo(entry, args), + "docker" => BuildDockerStartInfo(entry, targetDirectory, resultsDirectory, args), + _ => BuildCustomRunnerStartInfo(runner, entry, args) + }; + } + + private static ProcessStartInfo BuildNativeStartInfo(string binaryPath, IReadOnlyList args) + { + if (string.IsNullOrWhiteSpace(binaryPath) || !File.Exists(binaryPath)) + { + throw new FileNotFoundException("Scanner entrypoint not found.", binaryPath); + } + + var startInfo = new ProcessStartInfo + { + FileName = binaryPath, + WorkingDirectory = Directory.GetCurrentDirectory() + }; + + foreach (var argument in args) + { + startInfo.ArgumentList.Add(argument); + } + + startInfo.RedirectStandardError = true; + startInfo.RedirectStandardOutput = true; + startInfo.UseShellExecute = false; + + return startInfo; + } + + private static ProcessStartInfo BuildDotNetStartInfo(string binaryPath, IReadOnlyList args) + { + var startInfo = new ProcessStartInfo + { + FileName = "dotnet", + WorkingDirectory = Directory.GetCurrentDirectory() + }; + + startInfo.ArgumentList.Add(binaryPath); + foreach (var argument in args) + { + startInfo.ArgumentList.Add(argument); + } + + startInfo.RedirectStandardError = true; + startInfo.RedirectStandardOutput = true; + startInfo.UseShellExecute = false; + + return startInfo; + } + + private static ProcessStartInfo BuildDockerStartInfo(string image, string targetDirectory, string resultsDirectory, IReadOnlyList args) + { + if (string.IsNullOrWhiteSpace(image)) + { + throw new ArgumentException("Docker image must be provided when runner is 'docker'.", nameof(image)); + } + + var cwd = Directory.GetCurrentDirectory(); + + var startInfo = new ProcessStartInfo + { + FileName = "docker", + WorkingDirectory = cwd + }; + + startInfo.ArgumentList.Add("run"); + startInfo.ArgumentList.Add("--rm"); + startInfo.ArgumentList.Add("-v"); + startInfo.ArgumentList.Add($"{cwd}:{cwd}"); + startInfo.ArgumentList.Add("-v"); + startInfo.ArgumentList.Add($"{targetDirectory}:/scan-target:ro"); + startInfo.ArgumentList.Add("-v"); + startInfo.ArgumentList.Add($"{resultsDirectory}:/scan-results"); + startInfo.ArgumentList.Add("-w"); + startInfo.ArgumentList.Add(cwd); + startInfo.ArgumentList.Add(image); + startInfo.ArgumentList.Add("--target"); + startInfo.ArgumentList.Add("/scan-target"); + startInfo.ArgumentList.Add("--output"); + startInfo.ArgumentList.Add("/scan-results/scan.json"); + + foreach (var argument in args) + { + startInfo.ArgumentList.Add(argument); + } + + startInfo.RedirectStandardError = true; + startInfo.RedirectStandardOutput = true; + startInfo.UseShellExecute = false; + + return startInfo; + } + + private static ProcessStartInfo BuildCustomRunnerStartInfo(string runner, string entry, IReadOnlyList args) + { + var startInfo = new ProcessStartInfo + { + FileName = runner, + WorkingDirectory = Directory.GetCurrentDirectory() + }; + + if (!string.IsNullOrWhiteSpace(entry)) + { + startInfo.ArgumentList.Add(entry); + } + + foreach (var argument in args) + { + startInfo.ArgumentList.Add(argument); + } + + startInfo.RedirectStandardError = true; + startInfo.RedirectStandardOutput = true; + startInfo.UseShellExecute = false; + + return startInfo; + } + + private static string ResolveResultsPath(string resultsDirectory, DateTimeOffset startTimestamp, HashSet baseline) + { + var candidates = Directory.GetFiles(resultsDirectory, "*", SearchOption.AllDirectories); + string? newest = null; + DateTimeOffset newestTimestamp = startTimestamp; + + foreach (var candidate in candidates) + { + if (baseline.Contains(candidate)) + { + continue; + } + + var info = new FileInfo(candidate); + if (info.LastWriteTimeUtc >= newestTimestamp) + { + newestTimestamp = info.LastWriteTimeUtc; + newest = candidate; + } + } + + return newest ?? string.Empty; + } + + private static string CreatePlaceholderResult(string resultsDirectory) + { + var fileName = $"scan-{DateTimeOffset.UtcNow:yyyyMMddHHmmss}.json"; + var path = Path.Combine(resultsDirectory, fileName); + File.WriteAllText(path, "{\"status\":\"placeholder\"}"); + return path; + } + + private static string WriteRunMetadata( + string resultsDirectory, + DateTimeOffset startedAt, + DateTimeOffset completedAt, + string runner, + string entry, + string targetDirectory, + string resultsPath, + IReadOnlyList arguments, + int exitCode, + IReadOnlyList stdout, + IReadOnlyList stderr) + { + var duration = completedAt - startedAt; + var payload = new + { + runner, + entry, + targetDirectory, + resultsPath, + arguments, + exitCode, + startedAt = startedAt, + completedAt = completedAt, + durationSeconds = Math.Round(duration.TotalSeconds, 3, MidpointRounding.AwayFromZero), + stdout, + stderr + }; + + var fileName = $"scan-run-{startedAt:yyyyMMddHHmmssfff}.json"; + var path = Path.Combine(resultsDirectory, fileName); + var options = new JsonSerializerOptions + { + WriteIndented = true + }; + var json = JsonSerializer.Serialize(payload, options); + File.WriteAllText(path, json); + return path; + } +} diff --git a/src/StellaOps.Cli/Services/ScannerInstaller.cs b/src/StellaOps.Cli/Services/ScannerInstaller.cs new file mode 100644 index 00000000..673e94c8 --- /dev/null +++ b/src/StellaOps.Cli/Services/ScannerInstaller.cs @@ -0,0 +1,79 @@ +using System; +using System.Diagnostics; +using System.IO; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.Extensions.Logging; + +namespace StellaOps.Cli.Services; + +internal sealed class ScannerInstaller : IScannerInstaller +{ + private readonly ILogger _logger; + + public ScannerInstaller(ILogger logger) + { + _logger = logger ?? throw new ArgumentNullException(nameof(logger)); + } + + public async Task InstallAsync(string artifactPath, bool verbose, CancellationToken cancellationToken) + { + if (string.IsNullOrWhiteSpace(artifactPath) || !File.Exists(artifactPath)) + { + throw new FileNotFoundException("Scanner artifact not found for installation.", artifactPath); + } + + // Current implementation assumes docker-based scanner bundle. + var processInfo = new ProcessStartInfo + { + FileName = "docker", + ArgumentList = { "load", "-i", artifactPath }, + RedirectStandardOutput = true, + RedirectStandardError = true, + UseShellExecute = false + }; + + using var process = new Process { StartInfo = processInfo, EnableRaisingEvents = true }; + + process.OutputDataReceived += (_, args) => + { + if (args.Data is null) + { + return; + } + + if (verbose) + { + _logger.LogInformation("[install] {Line}", args.Data); + } + }; + + process.ErrorDataReceived += (_, args) => + { + if (args.Data is null) + { + return; + } + + _logger.LogError("[install] {Line}", args.Data); + }; + + _logger.LogInformation("Installing scanner container from {Path}...", artifactPath); + if (!process.Start()) + { + throw new InvalidOperationException("Failed to start container installation process."); + } + + process.BeginOutputReadLine(); + process.BeginErrorReadLine(); + + await process.WaitForExitAsync(cancellationToken).ConfigureAwait(false); + + if (process.ExitCode != 0) + { + throw new InvalidOperationException($"Container installation failed with exit code {process.ExitCode}."); + } + + _logger.LogInformation("Scanner container installed successfully."); + } +} diff --git a/src/StellaOps.Cli/StellaOps.Cli.csproj b/src/StellaOps.Cli/StellaOps.Cli.csproj new file mode 100644 index 00000000..0ca66b58 --- /dev/null +++ b/src/StellaOps.Cli/StellaOps.Cli.csproj @@ -0,0 +1,44 @@ + + + + Exe + net10.0 + enable + enable + + + + + + + + + + + + + + + + + + PreserveNewest + + + PreserveNewest + + + PreserveNewest + + + PreserveNewest + + + + + + + + + + diff --git a/src/StellaOps.Cli/TASKS.md b/src/StellaOps.Cli/TASKS.md new file mode 100644 index 00000000..47718e33 --- /dev/null +++ b/src/StellaOps.Cli/TASKS.md @@ -0,0 +1,14 @@ +# TASKS +| Task | Owner(s) | Depends on | Notes | +|---|---|---|---| +|Bootstrap configuration fallback (env → appsettings{{.json/.yaml}})|DevEx/CLI|Core|**DONE** – CLI loads `API_KEY`/`STELLAOPS_BACKEND_URL` from environment or local settings, defaulting to empty strings when unset.| +|Introduce command host & routing skeleton|DevEx/CLI|Configuration|**DONE** – System.CommandLine (v2.0.0-beta5) router stitched with `scanner`, `scan`, `db`, and `config` verbs.| +|Scanner artifact download/install commands|Ops Integrator|Backend contracts|**DONE** – `scanner download` caches bundles, validates SHA-256 (plus optional RSA signature), installs via `docker load`, persists metadata, and retries with exponential backoff.| +|Scan execution & result upload workflow|Ops Integrator, QA|Scanner cmd|**DONE** – `scan run` drives container scans against directories, emits artefacts in `ResultsDirectory`, auto-uploads on success, and `scan upload` covers manual retries.| +|Feedser DB operations passthrough|DevEx/CLI|Backend, Feedser APIs|**DONE** – `db fetch|merge|export` trigger `/jobs/*` endpoints with parameter binding and consistent exit codes.| +|CLI observability & tests|QA|Command host|**DONE** – Added console logging defaults & configuration bootstrap tests; future metrics hooks tracked separately.| +|Authority auth commands|DevEx/CLI|Auth libraries|**DONE** – `auth login/logout/status` wrap the shared auth client, manage token cache, and surface status messages.| +|Document authority workflow in CLI help & quickstart|Docs/CLI|Authority auth commands|**DONE (2025-10-10)** – CLI help now surfaces Authority config fields and docs/09 + docs/10 describe env vars, auth login/status flow, and cache location.| +|Authority whoami command|DevEx/CLI|Authority auth commands|**DONE (2025-10-10)** – Added `auth whoami` verb that displays subject/audience/expiry from cached tokens and handles opaque tokens gracefully.| +|Expose auth client resilience settings|DevEx/CLI|Auth libraries LIB5|**DONE (2025-10-10)** – CLI options now bind resilience knobs, `AddStellaOpsAuthClient` honours them, and tests cover env overrides.| +|Document advanced Authority tuning|Docs/CLI|Expose auth client resilience settings|**DONE (2025-10-10)** – docs/09 and docs/10 describe retry/offline settings with env examples and point to the integration guide.| diff --git a/src/StellaOps.Cli/Telemetry/CliActivitySource.cs b/src/StellaOps.Cli/Telemetry/CliActivitySource.cs new file mode 100644 index 00000000..7a2f16f8 --- /dev/null +++ b/src/StellaOps.Cli/Telemetry/CliActivitySource.cs @@ -0,0 +1,8 @@ +using System.Diagnostics; + +namespace StellaOps.Cli.Telemetry; + +internal static class CliActivitySource +{ + public static readonly ActivitySource Instance = new("StellaOps.Cli"); +} diff --git a/src/StellaOps.Cli/Telemetry/CliMetrics.cs b/src/StellaOps.Cli/Telemetry/CliMetrics.cs new file mode 100644 index 00000000..21206108 --- /dev/null +++ b/src/StellaOps.Cli/Telemetry/CliMetrics.cs @@ -0,0 +1,62 @@ +using System; +using System.Diagnostics.Metrics; + +namespace StellaOps.Cli.Telemetry; + +internal static class CliMetrics +{ + private static readonly Meter Meter = new("StellaOps.Cli", "1.0.0"); + + private static readonly Counter ScannerDownloadCounter = Meter.CreateCounter("stellaops.cli.scanner.download.count"); + private static readonly Counter ScannerInstallCounter = Meter.CreateCounter("stellaops.cli.scanner.install.count"); + private static readonly Counter ScanRunCounter = Meter.CreateCounter("stellaops.cli.scan.run.count"); + private static readonly Histogram CommandDurationHistogram = Meter.CreateHistogram("stellaops.cli.command.duration.ms"); + + public static void RecordScannerDownload(string channel, bool fromCache) + => ScannerDownloadCounter.Add(1, new KeyValuePair[] + { + new("channel", channel), + new("cache", fromCache ? "hit" : "miss") + }); + + public static void RecordScannerInstall(string channel) + => ScannerInstallCounter.Add(1, new KeyValuePair[] { new("channel", channel) }); + + public static void RecordScanRun(string runner, int exitCode) + => ScanRunCounter.Add(1, new KeyValuePair[] + { + new("runner", runner), + new("exit_code", exitCode) + }); + + public static IDisposable MeasureCommandDuration(string command) + { + var start = DateTime.UtcNow; + return new DurationScope(command, start); + } + + private sealed class DurationScope : IDisposable + { + private readonly string _command; + private readonly DateTime _start; + private bool _disposed; + + public DurationScope(string command, DateTime start) + { + _command = command; + _start = start; + } + + public void Dispose() + { + if (_disposed) + { + return; + } + + _disposed = true; + var elapsed = (DateTime.UtcNow - _start).TotalMilliseconds; + CommandDurationHistogram.Record(elapsed, new KeyValuePair[] { new("command", _command) }); + } + } +} diff --git a/src/StellaOps.Cli/Telemetry/VerbosityState.cs b/src/StellaOps.Cli/Telemetry/VerbosityState.cs new file mode 100644 index 00000000..d5973199 --- /dev/null +++ b/src/StellaOps.Cli/Telemetry/VerbosityState.cs @@ -0,0 +1,8 @@ +using Microsoft.Extensions.Logging; + +namespace StellaOps.Cli.Telemetry; + +internal sealed class VerbosityState +{ + public LogLevel MinimumLevel { get; set; } = LogLevel.Information; +} diff --git a/src/StellaOps.Cli/appsettings.json b/src/StellaOps.Cli/appsettings.json new file mode 100644 index 00000000..42283e98 --- /dev/null +++ b/src/StellaOps.Cli/appsettings.json @@ -0,0 +1,11 @@ +{ + "StellaOps": { + "ApiKey": "", + "BackendUrl": "", + "ScannerCacheDirectory": "scanners", + "ResultsDirectory": "results", + "DefaultRunner": "dotnet", + "ScannerSignaturePublicKeyPath": "", + "ScannerDownloadAttempts": 3 + } +} diff --git a/src/StellaOps.Configuration.Tests/AuthorityPluginConfigurationLoaderTests.cs b/src/StellaOps.Configuration.Tests/AuthorityPluginConfigurationLoaderTests.cs new file mode 100644 index 00000000..76bc24cd --- /dev/null +++ b/src/StellaOps.Configuration.Tests/AuthorityPluginConfigurationLoaderTests.cs @@ -0,0 +1,126 @@ +using System; +using System.IO; +using StellaOps.Authority.Plugins.Abstractions; +using StellaOps.Configuration; +using Xunit; + +namespace StellaOps.Configuration.Tests; + +public class AuthorityPluginConfigurationLoaderTests : IDisposable +{ + private readonly string tempRoot; + + public AuthorityPluginConfigurationLoaderTests() + { + tempRoot = Path.Combine(Path.GetTempPath(), "authority-plugin-tests", Guid.NewGuid().ToString("N")); + Directory.CreateDirectory(tempRoot); + } + + [Fact] + public void Load_ReturnsConfiguration_ForEnabledPlugin() + { + var pluginDir = Path.Combine(tempRoot, "etc", "authority.plugins"); + Directory.CreateDirectory(pluginDir); + + var standardConfigPath = Path.Combine(pluginDir, "standard.yaml"); + File.WriteAllText(standardConfigPath, "secretKey: value"); + + var options = CreateOptions(); + options.Plugins.ConfigurationDirectory = "etc/authority.plugins"; + options.Plugins.Descriptors["standard"] = new AuthorityPluginDescriptorOptions + { + AssemblyName = "StellaOps.Authority.Plugin.Standard", + Enabled = true + }; + + options.Validate(); + + var contexts = AuthorityPluginConfigurationLoader.Load(options, tempRoot); + var context = Assert.Single(contexts); + Assert.Equal("standard", context.Manifest.Name); + Assert.Equal("value", context.Configuration["secretKey"]); + Assert.True(context.Manifest.Enabled); + } + + [Fact] + public void Load_Throws_WhenEnabledConfigMissing() + { + var options = CreateOptions(); + options.Plugins.ConfigurationDirectory = "etc/authority.plugins"; + options.Plugins.Descriptors["standard"] = new AuthorityPluginDescriptorOptions + { + AssemblyName = "StellaOps.Authority.Plugin.Standard", + Enabled = true + }; + + options.Validate(); + + var ex = Assert.Throws(() => + AuthorityPluginConfigurationLoader.Load(options, tempRoot)); + + Assert.Contains("standard.yaml", ex.FileName, StringComparison.OrdinalIgnoreCase); + } + + [Fact] + public void Load_SkipsMissingFile_ForDisabledPlugin() + { + var options = CreateOptions(); + options.Plugins.ConfigurationDirectory = "etc/authority.plugins"; + options.Plugins.Descriptors["ldap"] = new AuthorityPluginDescriptorOptions + { + AssemblyName = "StellaOps.Authority.Plugin.Ldap", + Enabled = false, + ConfigFile = "ldap.yaml" + }; + + options.Validate(); + + var contexts = AuthorityPluginConfigurationLoader.Load(options, tempRoot); + var context = Assert.Single(contexts); + Assert.False(context.Manifest.Enabled); + Assert.Equal("ldap", context.Manifest.Name); + Assert.Null(context.Configuration["connection:host"]); + } + + [Fact] + public void Validate_ThrowsForUnknownCapability() + { + var options = CreateOptions(); + options.Plugins.Descriptors["standard"] = new AuthorityPluginDescriptorOptions + { + AssemblyName = "StellaOps.Authority.Plugin.Standard", + Enabled = true + }; + options.Plugins.Descriptors["standard"].Capabilities.Add("custom-flow"); + + var ex = Assert.Throws(() => options.Validate()); + Assert.Contains("unknown capability", ex.Message, StringComparison.OrdinalIgnoreCase); + } + + public void Dispose() + { + try + { + if (Directory.Exists(tempRoot)) + { + Directory.Delete(tempRoot, recursive: true); + } + } + catch + { + // ignore cleanup failures in test environment + } + } + + private static StellaOpsAuthorityOptions CreateOptions() + { + var options = new StellaOpsAuthorityOptions + { + Issuer = new Uri("https://authority.stella-ops.test"), + SchemaVersion = 1 + }; + + options.Storage.ConnectionString = "mongodb://localhost:27017/authority_test"; + return options; + } +} diff --git a/src/StellaOps.Configuration.Tests/AuthorityTelemetryTests.cs b/src/StellaOps.Configuration.Tests/AuthorityTelemetryTests.cs new file mode 100644 index 00000000..a567dc94 --- /dev/null +++ b/src/StellaOps.Configuration.Tests/AuthorityTelemetryTests.cs @@ -0,0 +1,24 @@ +using StellaOps.Auth; +using Xunit; + +namespace StellaOps.Configuration.Tests; + +public class AuthorityTelemetryTests +{ + [Fact] + public void ServiceName_AndNamespace_MatchExpectations() + { + Assert.Equal("stellaops-authority", AuthorityTelemetry.ServiceName); + Assert.Equal("stellaops", AuthorityTelemetry.ServiceNamespace); + } + + [Fact] + public void BuildDefaultResourceAttributes_ContainsExpectedKeys() + { + var attributes = AuthorityTelemetry.BuildDefaultResourceAttributes(); + + Assert.Equal("stellaops-authority", attributes["service.name"]); + Assert.Equal("stellaops", attributes["service.namespace"]); + Assert.False(string.IsNullOrWhiteSpace(attributes["service.version"]?.ToString())); + } +} diff --git a/src/StellaOps.Configuration.Tests/StellaOps.Configuration.Tests.csproj b/src/StellaOps.Configuration.Tests/StellaOps.Configuration.Tests.csproj new file mode 100644 index 00000000..346491bf --- /dev/null +++ b/src/StellaOps.Configuration.Tests/StellaOps.Configuration.Tests.csproj @@ -0,0 +1,11 @@ + + + net10.0 + enable + enable + + + + + + diff --git a/src/StellaOps.Configuration.Tests/StellaOpsAuthorityOptionsTests.cs b/src/StellaOps.Configuration.Tests/StellaOpsAuthorityOptionsTests.cs new file mode 100644 index 00000000..c3852e69 --- /dev/null +++ b/src/StellaOps.Configuration.Tests/StellaOpsAuthorityOptionsTests.cs @@ -0,0 +1,148 @@ +using System; +using System.Collections.Generic; +using Microsoft.Extensions.Configuration; +using StellaOps.Configuration; +using Xunit; + +namespace StellaOps.Configuration.Tests; + +public class StellaOpsAuthorityOptionsTests +{ + [Fact] + public void Validate_Throws_When_IssuerMissing() + { + var options = new StellaOpsAuthorityOptions(); + + var exception = Assert.Throws(() => options.Validate()); + + Assert.Contains("issuer", exception.Message, StringComparison.OrdinalIgnoreCase); + } + + [Fact] + public void Validate_Normalises_Collections() + { + var options = new StellaOpsAuthorityOptions + { + Issuer = new Uri("https://authority.stella-ops.test"), + SchemaVersion = 1 + }; + options.Storage.ConnectionString = "mongodb://localhost:27017/authority"; + + options.PluginDirectories.Add(" ./plugins "); + options.PluginDirectories.Add("./plugins"); + options.PluginDirectories.Add("./other"); + + options.BypassNetworks.Add(" 10.0.0.0/24 "); + options.BypassNetworks.Add("10.0.0.0/24"); + options.BypassNetworks.Add("192.168.0.0/16"); + + options.Validate(); + + Assert.Equal(new[] { "./plugins", "./other" }, options.PluginDirectories); + Assert.Equal(new[] { "10.0.0.0/24", "192.168.0.0/16" }, options.BypassNetworks); + } + + [Fact] + public void Validate_Normalises_PluginDescriptors() + { + var options = new StellaOpsAuthorityOptions + { + Issuer = new Uri("https://authority.stella-ops.test"), + SchemaVersion = 1 + }; + options.Storage.ConnectionString = "mongodb://localhost:27017/authority"; + + var descriptor = new AuthorityPluginDescriptorOptions + { + AssemblyName = "StellaOps.Authority.Plugin.Standard", + ConfigFile = " standard.yaml ", + Enabled = true + }; + + descriptor.Capabilities.Add("password"); + descriptor.Capabilities.Add("PASSWORD"); + options.Plugins.Descriptors["standard"] = descriptor; + + options.Validate(); + + var normalized = options.Plugins.Descriptors["standard"]; + Assert.Equal("standard.yaml", normalized.ConfigFile); + Assert.Single(normalized.Capabilities); + Assert.Equal("password", normalized.Capabilities[0]); + } + + [Fact] + public void Validate_Throws_When_StorageConnectionStringMissing() + { + var options = new StellaOpsAuthorityOptions + { + Issuer = new Uri("https://authority.stella-ops.test"), + SchemaVersion = 1 + }; + + var exception = Assert.Throws(() => options.Validate()); + + Assert.Contains("Mongo connection string", exception.Message, StringComparison.OrdinalIgnoreCase); + } + + [Fact] + public void Build_Binds_From_Configuration() + { + var context = StellaOpsAuthorityConfiguration.Build(options => + { + options.ConfigureBuilder = builder => + { + builder.AddInMemoryCollection(new Dictionary + { + ["Authority:SchemaVersion"] = "2", + ["Authority:Issuer"] = "https://authority.internal", + ["Authority:AccessTokenLifetime"] = "00:30:00", + ["Authority:RefreshTokenLifetime"] = "30.00:00:00", + ["Authority:Storage:ConnectionString"] = "mongodb://example/stellaops", + ["Authority:Storage:DatabaseName"] = "overrideDb", + ["Authority:Storage:CommandTimeout"] = "00:01:30", + ["Authority:PluginDirectories:0"] = "/var/lib/stellaops/plugins", + ["Authority:BypassNetworks:0"] = "127.0.0.1/32", + ["Authority:Security:RateLimiting:Token:PermitLimit"] = "25", + ["Authority:Security:RateLimiting:Token:Window"] = "00:00:30", + ["Authority:Security:RateLimiting:Authorize:Enabled"] = "true", + ["Authority:Security:RateLimiting:Internal:Enabled"] = "true", + ["Authority:Security:RateLimiting:Internal:PermitLimit"] = "3" + }); + }; + }); + + var options = context.Options; + + Assert.Equal(2, options.SchemaVersion); + Assert.Equal(new Uri("https://authority.internal"), options.Issuer); + Assert.Equal(TimeSpan.FromMinutes(30), options.AccessTokenLifetime); + Assert.Equal(TimeSpan.FromDays(30), options.RefreshTokenLifetime); + Assert.Equal(new[] { "/var/lib/stellaops/plugins" }, options.PluginDirectories); + Assert.Equal(new[] { "127.0.0.1/32" }, options.BypassNetworks); + Assert.Equal("mongodb://example/stellaops", options.Storage.ConnectionString); + Assert.Equal("overrideDb", options.Storage.DatabaseName); + Assert.Equal(TimeSpan.FromMinutes(1.5), options.Storage.CommandTimeout); + Assert.Equal(25, options.Security.RateLimiting.Token.PermitLimit); + Assert.Equal(TimeSpan.FromSeconds(30), options.Security.RateLimiting.Token.Window); + Assert.True(options.Security.RateLimiting.Authorize.Enabled); + Assert.True(options.Security.RateLimiting.Internal.Enabled); + Assert.Equal(3, options.Security.RateLimiting.Internal.PermitLimit); + } + + [Fact] + public void Validate_Throws_When_RateLimitingInvalid() + { + var options = new StellaOpsAuthorityOptions + { + Issuer = new Uri("https://authority.stella-ops.test"), + SchemaVersion = 1 + }; + options.Storage.ConnectionString = "mongodb://localhost:27017/authority"; + options.Security.RateLimiting.Token.PermitLimit = 0; + + var exception = Assert.Throws(() => options.Validate()); + + Assert.Contains("permitLimit", exception.Message, StringComparison.OrdinalIgnoreCase); + } +} diff --git a/src/StellaOps.Configuration/AuthorityPluginConfigurationLoader.cs b/src/StellaOps.Configuration/AuthorityPluginConfigurationLoader.cs new file mode 100644 index 00000000..ddd740db --- /dev/null +++ b/src/StellaOps.Configuration/AuthorityPluginConfigurationLoader.cs @@ -0,0 +1,100 @@ +using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using Microsoft.Extensions.Configuration; +using NetEscapades.Configuration.Yaml; +using StellaOps.Authority.Plugins.Abstractions; + +namespace StellaOps.Configuration; + +/// +/// Utility helpers for loading Authority plugin configuration manifests. +/// +public static class AuthorityPluginConfigurationLoader +{ + /// + /// Loads plugin configuration files based on the supplied Authority options. + /// + /// Authority configuration containing plugin descriptors. + /// Application base path used to resolve relative directories. + /// Optional hook to customise per-plugin configuration builder. + public static IReadOnlyList Load( + StellaOpsAuthorityOptions options, + string basePath, + Action? configureBuilder = null) + { + ArgumentNullException.ThrowIfNull(options); + ArgumentNullException.ThrowIfNull(basePath); + + var descriptorPairs = options.Plugins.Descriptors + .OrderBy(static pair => pair.Key, StringComparer.OrdinalIgnoreCase) + .ToArray(); + + if (descriptorPairs.Length == 0) + { + return Array.Empty(); + } + + var configurationDirectory = ResolveConfigurationDirectory(options.Plugins.ConfigurationDirectory, basePath); + var contexts = new List(descriptorPairs.Length); + + foreach (var (name, descriptor) in descriptorPairs) + { + var configPath = ResolveConfigPath(configurationDirectory, descriptor.ConfigFile); + var optional = !descriptor.Enabled; + + if (!optional && !File.Exists(configPath)) + { + throw new FileNotFoundException($"Required Authority plugin configuration '{configPath}' was not found.", configPath); + } + + var builder = new ConfigurationBuilder(); + var builderBasePath = Path.GetDirectoryName(configPath); + if (!string.IsNullOrEmpty(builderBasePath) && Directory.Exists(builderBasePath)) + { + builder.SetBasePath(builderBasePath); + } + + configureBuilder?.Invoke(builder); + builder.AddYamlFile(configPath, optional: optional, reloadOnChange: false); + var configuration = builder.Build(); + + var manifest = descriptor.ToManifest(name, configPath); + contexts.Add(new AuthorityPluginContext(manifest, configuration)); + } + + return contexts; + } + + private static string ResolveConfigurationDirectory(string configurationDirectory, string basePath) + { + if (string.IsNullOrWhiteSpace(configurationDirectory)) + { + return Path.GetFullPath(basePath); + } + + var directory = configurationDirectory; + if (!Path.IsPathRooted(directory)) + { + directory = Path.Combine(basePath, directory); + } + + return Path.GetFullPath(directory); + } + + private static string ResolveConfigPath(string configurationDirectory, string? configFile) + { + if (string.IsNullOrWhiteSpace(configFile)) + { + throw new InvalidOperationException("Authority plugin descriptor must specify a configFile."); + } + + if (Path.IsPathRooted(configFile)) + { + return Path.GetFullPath(configFile); + } + + return Path.GetFullPath(Path.Combine(configurationDirectory, configFile)); + } +} diff --git a/src/StellaOps.Configuration/StellaOps.Configuration.csproj b/src/StellaOps.Configuration/StellaOps.Configuration.csproj new file mode 100644 index 00000000..af97e2fd --- /dev/null +++ b/src/StellaOps.Configuration/StellaOps.Configuration.csproj @@ -0,0 +1,23 @@ + + + + net10.0 + enable + enable + + + + + + + + + + + + + + + + + diff --git a/src/StellaOps.Configuration/StellaOpsAuthorityConfiguration.cs b/src/StellaOps.Configuration/StellaOpsAuthorityConfiguration.cs new file mode 100644 index 00000000..8d93baa7 --- /dev/null +++ b/src/StellaOps.Configuration/StellaOpsAuthorityConfiguration.cs @@ -0,0 +1,57 @@ +using System; +using System.Linq; + +namespace StellaOps.Configuration; + +/// +/// Helper utilities for bootstrapping StellaOps Authority configuration. +/// +public static class StellaOpsAuthorityConfiguration +{ + private static readonly string[] DefaultAuthorityYamlFiles = + { + "authority.yaml", + "authority.local.yaml", + "etc/authority.yaml", + "etc/authority.local.yaml" + }; + + /// + /// Builds using the shared configuration bootstrapper. + /// + /// Optional hook to customise bootstrap behaviour. + public static StellaOpsConfigurationContext Build( + Action>? configure = null) + { + return StellaOpsConfigurationBootstrapper.Build(options => + { + options.BindingSection ??= "Authority"; + options.EnvironmentPrefix ??= "STELLAOPS_AUTHORITY_"; + + configure?.Invoke(options); + + AppendDefaultYamlFiles(options); + + var previousPostBind = options.PostBind; + options.PostBind = (authorityOptions, configuration) => + { + previousPostBind?.Invoke(authorityOptions, configuration); + authorityOptions.Validate(); + }; + }); + } + + private static void AppendDefaultYamlFiles(StellaOpsBootstrapOptions options) + { + foreach (var path in DefaultAuthorityYamlFiles) + { + var alreadyPresent = options.YamlFiles.Any(file => + string.Equals(file.Path, path, StringComparison.OrdinalIgnoreCase)); + + if (!alreadyPresent) + { + options.YamlFiles.Add(new YamlConfigurationFile(path, Optional: true)); + } + } + } +} diff --git a/src/StellaOps.Configuration/StellaOpsAuthorityOptions.cs b/src/StellaOps.Configuration/StellaOpsAuthorityOptions.cs new file mode 100644 index 00000000..7d466431 --- /dev/null +++ b/src/StellaOps.Configuration/StellaOpsAuthorityOptions.cs @@ -0,0 +1,529 @@ +using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using System.Threading.RateLimiting; +using StellaOps.Authority.Plugins.Abstractions; + +namespace StellaOps.Configuration; + +/// +/// Strongly typed configuration for the StellaOps Authority service. +/// +public sealed class StellaOpsAuthorityOptions +{ + private readonly List pluginDirectories = new(); + private readonly List bypassNetworks = new(); + + /// + /// Schema version for downstream consumers to coordinate breaking changes. + /// + public int SchemaVersion { get; set; } = 1; + + /// + /// Absolute issuer URI advertised to clients (e.g. https://authority.stella-ops.local). + /// + public Uri? Issuer { get; set; } + + /// + /// Lifetime for OAuth access tokens issued by Authority. + /// + public TimeSpan AccessTokenLifetime { get; set; } = TimeSpan.FromMinutes(15); + + /// + /// Lifetime for OAuth refresh tokens issued by Authority. + /// + public TimeSpan RefreshTokenLifetime { get; set; } = TimeSpan.FromDays(30); + + /// + /// Lifetime for OpenID Connect identity tokens. + /// + public TimeSpan IdentityTokenLifetime { get; set; } = TimeSpan.FromMinutes(5); + + /// + /// Lifetime for OAuth authorization codes. + /// + public TimeSpan AuthorizationCodeLifetime { get; set; } = TimeSpan.FromMinutes(5); + + /// + /// Lifetime for OAuth device codes (device authorization flow). + /// + public TimeSpan DeviceCodeLifetime { get; set; } = TimeSpan.FromMinutes(15); + + /// + /// Directories scanned for Authority plugins (absolute or relative to application base path). + /// + public IList PluginDirectories => pluginDirectories; + + /// + /// CIDR blocks permitted to bypass certain authentication policies (e.g. on-host cron). + /// + public IList BypassNetworks => bypassNetworks; + + /// + /// Configuration describing the Authority MongoDB storage. + /// + public AuthorityStorageOptions Storage { get; } = new(); + + /// + /// Bootstrap settings for initial administrative provisioning. + /// + public AuthorityBootstrapOptions Bootstrap { get; } = new(); + + /// + /// Configuration describing available Authority plugins and their manifests. + /// + public AuthorityPluginSettings Plugins { get; } = new(); + + /// + /// Security-related configuration for the Authority host. + /// + public AuthoritySecurityOptions Security { get; } = new(); + + /// + /// Validates configured values and normalises collections. + /// + /// Thrown when configuration is invalid. + public void Validate() + { + if (SchemaVersion <= 0) + { + throw new InvalidOperationException("Authority configuration requires a positive schemaVersion."); + } + + if (Issuer is null) + { + throw new InvalidOperationException("Authority configuration requires an issuer URL."); + } + + if (!Issuer.IsAbsoluteUri) + { + throw new InvalidOperationException("Authority issuer must be an absolute URI."); + } + + if (string.Equals(Issuer.Scheme, Uri.UriSchemeHttp, StringComparison.OrdinalIgnoreCase) && !Issuer.IsLoopback) + { + throw new InvalidOperationException("Authority issuer must use HTTPS unless running on a loopback interface."); + } + + ValidateLifetime(AccessTokenLifetime, nameof(AccessTokenLifetime), TimeSpan.FromHours(24)); + ValidateLifetime(RefreshTokenLifetime, nameof(RefreshTokenLifetime), TimeSpan.FromDays(365)); + ValidateLifetime(IdentityTokenLifetime, nameof(IdentityTokenLifetime), TimeSpan.FromHours(24)); + ValidateLifetime(AuthorizationCodeLifetime, nameof(AuthorizationCodeLifetime), TimeSpan.FromHours(1)); + ValidateLifetime(DeviceCodeLifetime, nameof(DeviceCodeLifetime), TimeSpan.FromHours(24)); + + NormaliseList(pluginDirectories); + NormaliseList(bypassNetworks); + + Security.Validate(); + Plugins.NormalizeAndValidate(); + Storage.Validate(); + Bootstrap.Validate(); + } + + private static void ValidateLifetime(TimeSpan value, string propertyName, TimeSpan maximum) + { + if (value <= TimeSpan.Zero) + { + throw new InvalidOperationException($"Authority configuration requires {propertyName} to be greater than zero."); + } + + if (value > maximum) + { + throw new InvalidOperationException($"Authority configuration requires {propertyName} to be less than or equal to {maximum}."); + } + } + + private static void NormaliseList(IList values) + { + if (values.Count == 0) + { + return; + } + + var unique = new HashSet(StringComparer.OrdinalIgnoreCase); + + for (var index = values.Count - 1; index >= 0; index--) + { + var entry = values[index]; + + if (string.IsNullOrWhiteSpace(entry)) + { + values.RemoveAt(index); + continue; + } + + var trimmed = entry.Trim(); + if (!unique.Add(trimmed)) + { + values.RemoveAt(index); + continue; + } + + values[index] = trimmed; + } + } +} + +public sealed class AuthoritySecurityOptions +{ + /// + /// Rate limiting configuration applied to Authority endpoints. + /// + public AuthorityRateLimitingOptions RateLimiting { get; } = new(); + + internal void Validate() + { + RateLimiting.Validate(); + } +} + +public sealed class AuthorityRateLimitingOptions +{ + public AuthorityRateLimitingOptions() + { + Token = new AuthorityEndpointRateLimitOptions + { + PermitLimit = 30, + Window = TimeSpan.FromMinutes(1), + QueueLimit = 0 + }; + + Authorize = new AuthorityEndpointRateLimitOptions + { + PermitLimit = 60, + Window = TimeSpan.FromMinutes(1), + QueueLimit = 10 + }; + + Internal = new AuthorityEndpointRateLimitOptions + { + Enabled = false, + PermitLimit = 5, + Window = TimeSpan.FromMinutes(1), + QueueLimit = 0 + }; + } + + /// + /// Rate limiting configuration applied to the /token endpoint. + /// + public AuthorityEndpointRateLimitOptions Token { get; } + + /// + /// Rate limiting configuration applied to the /authorize endpoint. + /// + public AuthorityEndpointRateLimitOptions Authorize { get; } + + /// + /// Rate limiting configuration applied to /internal endpoints. + /// + public AuthorityEndpointRateLimitOptions Internal { get; } + + internal void Validate() + { + Token.Validate(nameof(Token)); + Authorize.Validate(nameof(Authorize)); + Internal.Validate(nameof(Internal)); + } +} + +public sealed class AuthorityEndpointRateLimitOptions +{ + /// + /// Gets or sets a value indicating whether rate limiting is enabled for the endpoint. + /// + public bool Enabled { get; set; } = true; + + /// + /// Maximum number of requests allowed within the configured window. + /// + public int PermitLimit { get; set; } = 60; + + /// + /// Size of the fixed window applied to the rate limiter. + /// + public TimeSpan Window { get; set; } = TimeSpan.FromMinutes(1); + + /// + /// Maximum number of queued requests awaiting permits. + /// + public int QueueLimit { get; set; } = 0; + + /// + /// Ordering strategy for queued requests. + /// + public QueueProcessingOrder QueueProcessingOrder { get; set; } = QueueProcessingOrder.OldestFirst; + + internal void Validate(string name) + { + if (!Enabled) + { + return; + } + + if (PermitLimit <= 0) + { + throw new InvalidOperationException($"Authority rate limiting '{name}' requires permitLimit to be greater than zero."); + } + + if (QueueLimit < 0) + { + throw new InvalidOperationException($"Authority rate limiting '{name}' queueLimit cannot be negative."); + } + + if (Window <= TimeSpan.Zero || Window > TimeSpan.FromHours(1)) + { + throw new InvalidOperationException($"Authority rate limiting '{name}' window must be greater than zero and no more than one hour."); + } + } +} + +public sealed class AuthorityStorageOptions +{ + /// + /// Mongo connection string used by Authority storage. + /// + public string ConnectionString { get; set; } = string.Empty; + + /// + /// Optional explicit database name override. + /// + public string? DatabaseName { get; set; } + + /// + /// Mongo command timeout. + /// + public TimeSpan CommandTimeout { get; set; } = TimeSpan.FromSeconds(30); + + internal void Validate() + { + if (string.IsNullOrWhiteSpace(ConnectionString)) + { + throw new InvalidOperationException("Authority storage requires a Mongo connection string."); + } + + if (CommandTimeout <= TimeSpan.Zero) + { + throw new InvalidOperationException("Authority storage command timeout must be greater than zero."); + } + } +} + +public sealed class AuthorityBootstrapOptions +{ + /// + /// Enables or disables bootstrap administrative APIs. + /// + public bool Enabled { get; set; } = false; + + /// + /// API key required when invoking bootstrap endpoints. + /// + public string? ApiKey { get; set; } = string.Empty; + + /// + /// Default identity provider used when none is specified in bootstrap requests. + /// + public string? DefaultIdentityProvider { get; set; } = "standard"; + + internal void Validate() + { + if (!Enabled) + { + return; + } + + if (string.IsNullOrWhiteSpace(ApiKey)) + { + throw new InvalidOperationException("Authority bootstrap configuration requires an API key when enabled."); + } + + if (string.IsNullOrWhiteSpace(DefaultIdentityProvider)) + { + throw new InvalidOperationException("Authority bootstrap configuration requires a default identity provider name when enabled."); + } + } +} + +public sealed class AuthorityPluginSettings +{ + private static readonly StringComparer OrdinalIgnoreCase = StringComparer.OrdinalIgnoreCase; + + /// + /// Directory containing per-plugin configuration manifests (relative paths resolved against application base path). + /// + public string ConfigurationDirectory { get; set; } = "../etc/authority.plugins"; + + /// + /// Declarative descriptors for Authority plugins (keyed by logical plugin name). + /// + public IDictionary Descriptors { get; } = new Dictionary(OrdinalIgnoreCase); + + internal void NormalizeAndValidate() + { + if (Descriptors.Count == 0) + { + return; + } + + foreach (var (name, descriptor) in Descriptors.ToArray()) + { + if (descriptor is null) + { + throw new InvalidOperationException($"Authority plugin descriptor '{name}' is null."); + } + + descriptor.Normalize(name); + descriptor.Validate(name); + } + } +} + +public sealed class AuthorityPluginDescriptorOptions +{ + private static readonly StringComparer OrdinalIgnoreCase = StringComparer.OrdinalIgnoreCase; + + private readonly List capabilities = new(); + private readonly Dictionary metadata = new(OrdinalIgnoreCase); + private static readonly HashSet AllowedCapabilities = new( + new[] + { + AuthorityPluginCapabilities.Password, + AuthorityPluginCapabilities.Mfa, + AuthorityPluginCapabilities.ClientProvisioning, + AuthorityPluginCapabilities.Bootstrap + }, + OrdinalIgnoreCase); + + /// + /// Logical type identifier for the plugin (e.g. standard, ldap). + /// + public string? Type { get; set; } + + /// + /// Name of the plugin assembly (without file extension). + /// + public string? AssemblyName { get; set; } + + /// + /// Optional explicit assembly path override; relative paths resolve against plugin directories. + /// + public string? AssemblyPath { get; set; } + + /// + /// Indicates whether the plugin should be enabled. + /// + public bool Enabled { get; set; } = true; + + /// + /// Plugin capability hints surfaced to the Authority host. + /// + public IList Capabilities => capabilities; + + /// + /// Optional metadata (string key/value) passed to plugin implementations. + /// + public IDictionary Metadata => metadata; + + /// + /// Relative path to the plugin-specific configuration file (defaults to <pluginName>.yaml). + /// + public string? ConfigFile { get; set; } + + internal void Normalize(string pluginName) + { + if (string.IsNullOrWhiteSpace(ConfigFile)) + { + ConfigFile = $"{pluginName}.yaml"; + } + else + { + ConfigFile = ConfigFile.Trim(); + } + + Type = string.IsNullOrWhiteSpace(Type) ? pluginName : Type.Trim(); + + if (!string.IsNullOrWhiteSpace(AssemblyName)) + { + AssemblyName = AssemblyName.Trim(); + } + + if (!string.IsNullOrWhiteSpace(AssemblyPath)) + { + AssemblyPath = AssemblyPath.Trim(); + } + + if (capabilities.Count > 0) + { + var seen = new HashSet(OrdinalIgnoreCase); + var unique = new List(capabilities.Count); + + foreach (var entry in capabilities) + { + if (string.IsNullOrWhiteSpace(entry)) + { + continue; + } + + var canonical = entry.Trim().ToLowerInvariant(); + if (seen.Add(canonical)) + { + unique.Add(canonical); + } + } + + unique.Sort(StringComparer.Ordinal); + + capabilities.Clear(); + capabilities.AddRange(unique); + } + } + + internal void Validate(string pluginName) + { + if (string.IsNullOrWhiteSpace(AssemblyName) && string.IsNullOrWhiteSpace(AssemblyPath)) + { + throw new InvalidOperationException($"Authority plugin '{pluginName}' must define either assemblyName or assemblyPath."); + } + + if (string.IsNullOrWhiteSpace(ConfigFile)) + { + throw new InvalidOperationException($"Authority plugin '{pluginName}' must define a configFile."); + } + + if (Path.GetFileName(ConfigFile) != ConfigFile && Path.IsPathRooted(ConfigFile) && !File.Exists(ConfigFile)) + { + throw new InvalidOperationException($"Authority plugin '{pluginName}' specifies configFile '{ConfigFile}' which does not exist."); + } + + foreach (var capability in capabilities) + { + if (!AllowedCapabilities.Contains(capability)) + { + throw new InvalidOperationException($"Authority plugin '{pluginName}' declares unknown capability '{capability}'. Allowed values: password, mfa, clientProvisioning, bootstrap."); + } + } + } + + internal AuthorityPluginManifest ToManifest(string name, string configPath) + { + var capabilitiesSnapshot = capabilities.Count == 0 + ? Array.Empty() + : capabilities.ToArray(); + + var metadataSnapshot = metadata.Count == 0 + ? new Dictionary(OrdinalIgnoreCase) + : new Dictionary(metadata, OrdinalIgnoreCase); + + return new AuthorityPluginManifest( + name, + Type ?? name, + Enabled, + AssemblyName, + AssemblyPath, + capabilitiesSnapshot, + metadataSnapshot, + configPath); + } +} diff --git a/src/StellaOps.Configuration/StellaOpsBootstrapOptions.cs b/src/StellaOps.Configuration/StellaOpsBootstrapOptions.cs new file mode 100644 index 00000000..78685657 --- /dev/null +++ b/src/StellaOps.Configuration/StellaOpsBootstrapOptions.cs @@ -0,0 +1,64 @@ +using System; +using System.Collections.Generic; +using Microsoft.Extensions.Configuration; + +namespace StellaOps.Configuration; + +public sealed class StellaOpsBootstrapOptions + where TOptions : class, new() +{ + public StellaOpsBootstrapOptions() + { + ConfigurationOptions = new StellaOpsConfigurationOptions(); + } + + internal StellaOpsConfigurationOptions ConfigurationOptions { get; } + + public string? BasePath + { + get => ConfigurationOptions.BasePath; + set => ConfigurationOptions.BasePath = value; + } + + public bool IncludeJsonFiles + { + get => ConfigurationOptions.IncludeJsonFiles; + set => ConfigurationOptions.IncludeJsonFiles = value; + } + + public bool IncludeYamlFiles + { + get => ConfigurationOptions.IncludeYamlFiles; + set => ConfigurationOptions.IncludeYamlFiles = value; + } + + public bool IncludeEnvironmentVariables + { + get => ConfigurationOptions.IncludeEnvironmentVariables; + set => ConfigurationOptions.IncludeEnvironmentVariables = value; + } + + public string? EnvironmentPrefix + { + get => ConfigurationOptions.EnvironmentPrefix; + set => ConfigurationOptions.EnvironmentPrefix = value; + } + + public IList JsonFiles => ConfigurationOptions.JsonFiles; + + public IList YamlFiles => ConfigurationOptions.YamlFiles; + + public string? BindingSection + { + get => ConfigurationOptions.BindingSection; + set => ConfigurationOptions.BindingSection = value; + } + + public Action? ConfigureBuilder + { + get => ConfigurationOptions.ConfigureBuilder; + set => ConfigurationOptions.ConfigureBuilder = value; + } + + public Action? PostBind { get; set; } +} diff --git a/src/StellaOps.Configuration/StellaOpsConfigurationBootstrapper.cs b/src/StellaOps.Configuration/StellaOpsConfigurationBootstrapper.cs new file mode 100644 index 00000000..84f81707 --- /dev/null +++ b/src/StellaOps.Configuration/StellaOpsConfigurationBootstrapper.cs @@ -0,0 +1,106 @@ +using System; +using Microsoft.Extensions.Configuration; +using NetEscapades.Configuration.Yaml; + +namespace StellaOps.Configuration; + +public static class StellaOpsConfigurationBootstrapper +{ + public static StellaOpsConfigurationContext Build( + Action>? configure = null) + where TOptions : class, new() + { + var bootstrapOptions = new StellaOpsBootstrapOptions(); + configure?.Invoke(bootstrapOptions); + + var configurationOptions = bootstrapOptions.ConfigurationOptions; + var builder = new ConfigurationBuilder(); + + if (!string.IsNullOrWhiteSpace(configurationOptions.BasePath)) + { + builder.SetBasePath(configurationOptions.BasePath!); + } + + if (configurationOptions.IncludeJsonFiles) + { + foreach (var file in configurationOptions.JsonFiles) + { + builder.AddJsonFile(file.Path, optional: file.Optional, reloadOnChange: file.ReloadOnChange); + } + } + + if (configurationOptions.IncludeYamlFiles) + { + foreach (var file in configurationOptions.YamlFiles) + { + builder.AddYamlFile(file.Path, optional: file.Optional); + } + } + + configurationOptions.ConfigureBuilder?.Invoke(builder); + + if (configurationOptions.IncludeEnvironmentVariables) + { + builder.AddEnvironmentVariables(configurationOptions.EnvironmentPrefix); + } + + var configuration = builder.Build(); + + IConfiguration bindingSource; + if (string.IsNullOrWhiteSpace(configurationOptions.BindingSection)) + { + bindingSource = configuration; + } + else + { + bindingSource = configuration.GetSection(configurationOptions.BindingSection!); + } + + var options = new TOptions(); + bindingSource.Bind(options); + + bootstrapOptions.PostBind?.Invoke(options, configuration); + + return new StellaOpsConfigurationContext(configuration, options); + } + + public static IConfigurationBuilder AddStellaOpsDefaults( + this IConfigurationBuilder builder, + Action? configure = null) + { + ArgumentNullException.ThrowIfNull(builder); + + var options = new StellaOpsConfigurationOptions(); + configure?.Invoke(options); + + if (!string.IsNullOrWhiteSpace(options.BasePath)) + { + builder.SetBasePath(options.BasePath!); + } + + if (options.IncludeJsonFiles) + { + foreach (var file in options.JsonFiles) + { + builder.AddJsonFile(file.Path, optional: file.Optional, reloadOnChange: file.ReloadOnChange); + } + } + + if (options.IncludeYamlFiles) + { + foreach (var file in options.YamlFiles) + { + builder.AddYamlFile(file.Path, optional: file.Optional); + } + } + + options.ConfigureBuilder?.Invoke(builder); + + if (options.IncludeEnvironmentVariables) + { + builder.AddEnvironmentVariables(options.EnvironmentPrefix); + } + + return builder; + } +} diff --git a/src/StellaOps.Configuration/StellaOpsConfigurationContext.cs b/src/StellaOps.Configuration/StellaOpsConfigurationContext.cs new file mode 100644 index 00000000..180a8fb2 --- /dev/null +++ b/src/StellaOps.Configuration/StellaOpsConfigurationContext.cs @@ -0,0 +1,18 @@ +using System; +using Microsoft.Extensions.Configuration; + +namespace StellaOps.Configuration; + +public sealed class StellaOpsConfigurationContext + where TOptions : class, new() +{ + public StellaOpsConfigurationContext(IConfigurationRoot configuration, TOptions options) + { + Configuration = configuration ?? throw new ArgumentNullException(nameof(configuration)); + Options = options ?? throw new ArgumentNullException(nameof(options)); + } + + public IConfigurationRoot Configuration { get; } + + public TOptions Options { get; } +} diff --git a/src/StellaOps.Configuration/StellaOpsConfigurationOptions.cs b/src/StellaOps.Configuration/StellaOpsConfigurationOptions.cs new file mode 100644 index 00000000..d1494c96 --- /dev/null +++ b/src/StellaOps.Configuration/StellaOpsConfigurationOptions.cs @@ -0,0 +1,49 @@ +using System; +using System.Collections.Generic; +using System.IO; +using Microsoft.Extensions.Configuration; + +namespace StellaOps.Configuration; + +/// +/// Defines how default StellaOps configuration sources are composed. +/// +public sealed class StellaOpsConfigurationOptions +{ + public string? BasePath { get; set; } = Directory.GetCurrentDirectory(); + + public bool IncludeJsonFiles { get; set; } = true; + + public bool IncludeYamlFiles { get; set; } = true; + + public bool IncludeEnvironmentVariables { get; set; } = true; + + public string? EnvironmentPrefix { get; set; } + + public IList JsonFiles { get; } = new List + { + new("appsettings.json", true, false), + new("appsettings.local.json", true, false) + }; + + public IList YamlFiles { get; } = new List + { + new("appsettings.yaml", true), + new("appsettings.local.yaml", true) + }; + + /// + /// Optional hook to register additional configuration sources (e.g. module-specific YAML files). + /// + public Action? ConfigureBuilder { get; set; } + + /// + /// Optional configuration section name used when binding strongly typed options. + /// Null or empty indicates the root. + /// + public string? BindingSection { get; set; } +} + +public sealed record JsonConfigurationFile(string Path, bool Optional = true, bool ReloadOnChange = false); + +public sealed record YamlConfigurationFile(string Path, bool Optional = true); diff --git a/src/StellaOps.Configuration/StellaOpsOptionsBinder.cs b/src/StellaOps.Configuration/StellaOpsOptionsBinder.cs new file mode 100644 index 00000000..5654c988 --- /dev/null +++ b/src/StellaOps.Configuration/StellaOpsOptionsBinder.cs @@ -0,0 +1,26 @@ +using System; +using Microsoft.Extensions.Configuration; + +namespace StellaOps.Configuration; + +public static class StellaOpsOptionsBinder +{ + public static TOptions BindOptions( + this IConfiguration configuration, + string? section = null, + Action? postConfigure = null) + where TOptions : class, new() + { + ArgumentNullException.ThrowIfNull(configuration); + + var options = new TOptions(); + var bindingSource = string.IsNullOrWhiteSpace(section) + ? configuration + : configuration.GetSection(section); + + bindingSource.Bind(options); + postConfigure?.Invoke(options, configuration); + + return options; + } +} diff --git a/src/StellaOps.Cryptography.Tests/PasswordHashOptionsTests.cs b/src/StellaOps.Cryptography.Tests/PasswordHashOptionsTests.cs new file mode 100644 index 00000000..f0187dcc --- /dev/null +++ b/src/StellaOps.Cryptography.Tests/PasswordHashOptionsTests.cs @@ -0,0 +1,24 @@ +using StellaOps.Cryptography; + +namespace StellaOps.Cryptography.Tests; + +public class PasswordHashOptionsTests +{ + [Fact] + public void Validate_DoesNotThrow_ForDefaults() + { + var options = new PasswordHashOptions(); + options.Validate(); + } + + [Fact] + public void Validate_Throws_WhenMemoryInvalid() + { + var options = new PasswordHashOptions + { + MemorySizeInKib = 0 + }; + + Assert.Throws(options.Validate); + } +} diff --git a/src/StellaOps.Cryptography.Tests/StellaOps.Cryptography.Tests.csproj b/src/StellaOps.Cryptography.Tests/StellaOps.Cryptography.Tests.csproj new file mode 100644 index 00000000..d3c903d3 --- /dev/null +++ b/src/StellaOps.Cryptography.Tests/StellaOps.Cryptography.Tests.csproj @@ -0,0 +1,11 @@ + + + net10.0 + enable + enable + false + + + + + diff --git a/src/StellaOps.Cryptography/AGENTS.md b/src/StellaOps.Cryptography/AGENTS.md new file mode 100644 index 00000000..b1124feb --- /dev/null +++ b/src/StellaOps.Cryptography/AGENTS.md @@ -0,0 +1,21 @@ +# Team 8 — Security Guild (Authority & Shared Crypto) + +## Role + +Team 8 owns the end-to-end security posture for StellaOps Authority and its consumers. That includes password hashing policy, audit/event hygiene, rate-limit & lockout rules, revocation distribution, and sovereign cryptography abstractions that allow alternative algorithm suites (e.g., GOST) without touching feature code. + +## Operational Boundaries + +- Primary workspace: `src/StellaOps.Cryptography`, `src/StellaOps.Authority.Plugin.Standard`, `src/StellaOps.Authority.Storage.Mongo`, and Authority host (`src/StellaOps.Authority/StellaOps.Authority`). +- Coordinate cross-module changes via TASKS.md updates and PR descriptions. +- Never bypass deterministic behaviour (sorted keys, stable timestamps). +- Tests live alongside owning projects (`*.Tests`). Extend goldens instead of rewriting. + +## Expectations + +- Default to Argon2id (Konscious) for password hashing; PBKDF2 only for legacy verification with transparent rehash on success. +- Emit structured security events with minimal PII and clear correlation IDs. +- Rate-limit `/token` and bootstrap endpoints once CORE8 hooks are available. +- Deliver offline revocation bundles signed with detached JWS and provide a verification script. +- Maintain `docs/security/authority-threat-model.md` and ensure mitigations are tracked. +- All crypto consumption flows through `StellaOps.Cryptography` abstractions to enable sovereign crypto providers. diff --git a/src/StellaOps.Cryptography/CryptoProvider.cs b/src/StellaOps.Cryptography/CryptoProvider.cs new file mode 100644 index 00000000..77403920 --- /dev/null +++ b/src/StellaOps.Cryptography/CryptoProvider.cs @@ -0,0 +1,44 @@ +using System.Collections.Generic; + +namespace StellaOps.Cryptography; + +/// +/// High-level cryptographic capabilities supported by StellaOps providers. +/// +public enum CryptoCapability +{ + PasswordHashing, + Signing, + Verification, + SymmetricEncryption, + KeyDerivation +} + +/// +/// Identifies a stored key or certificate handle. +/// +public sealed record CryptoKeyReference(string KeyId, string? ProviderHint = null); + +/// +/// Contract implemented by crypto providers (BCL, CryptoPro, OpenSSL, etc.). +/// +public interface ICryptoProvider +{ + string Name { get; } + + bool Supports(CryptoCapability capability, string algorithmId); + + IPasswordHasher GetPasswordHasher(string algorithmId); +} + +/// +/// Registry managing provider discovery and policy selection. +/// +public interface ICryptoProviderRegistry +{ + IReadOnlyCollection Providers { get; } + + bool TryResolve(string preferredProvider, out ICryptoProvider provider); + + ICryptoProvider ResolveOrThrow(CryptoCapability capability, string algorithmId); +} diff --git a/src/StellaOps.Cryptography/PasswordHashing.cs b/src/StellaOps.Cryptography/PasswordHashing.cs new file mode 100644 index 00000000..34feddba --- /dev/null +++ b/src/StellaOps.Cryptography/PasswordHashing.cs @@ -0,0 +1,81 @@ +using System; + +namespace StellaOps.Cryptography; + +/// +/// Supported password hashing algorithms. +/// +public enum PasswordHashAlgorithm +{ + Argon2id, + Pbkdf2 +} + +/// +/// Options describing password hashing requirements. +/// Values follow OWASP baseline guidance by default. +/// +public sealed record PasswordHashOptions +{ + /// + /// Algorithm to use when hashing new passwords. + /// + public PasswordHashAlgorithm Algorithm { get; init; } = PasswordHashAlgorithm.Argon2id; + + /// + /// Memory cost in KiB (default 19 MiB). + /// + public int MemorySizeInKib { get; init; } = 19 * 1024; + + /// + /// Iteration count / time cost. + /// + public int Iterations { get; init; } = 2; + + /// + /// Parallelism / degree of concurrency. + /// + public int Parallelism { get; init; } = 1; + + /// + /// Validates the option values and throws when invalid. + /// + public void Validate() + { + if (MemorySizeInKib <= 0) + { + throw new InvalidOperationException("Password hashing memory cost must be greater than zero."); + } + + if (Iterations <= 0) + { + throw new InvalidOperationException("Password hashing iteration count must be greater than zero."); + } + + if (Parallelism <= 0) + { + throw new InvalidOperationException("Password hashing parallelism must be greater than zero."); + } + } +} + +/// +/// Abstraction for password hashing implementations. +/// +public interface IPasswordHasher +{ + /// + /// Produces an encoded hash for the supplied password. + /// + string Hash(string password, PasswordHashOptions options); + + /// + /// Verifies the supplied password against a stored hash. + /// + bool Verify(string password, string encodedHash); + + /// + /// Detects when an existing encoded hash no longer satisfies the desired options. + /// + bool NeedsRehash(string encodedHash, PasswordHashOptions desired); +} diff --git a/src/StellaOps.Cryptography/StellaOps.Cryptography.csproj b/src/StellaOps.Cryptography/StellaOps.Cryptography.csproj new file mode 100644 index 00000000..514869b9 --- /dev/null +++ b/src/StellaOps.Cryptography/StellaOps.Cryptography.csproj @@ -0,0 +1,9 @@ + + + net10.0 + preview + enable + enable + true + + diff --git a/src/StellaOps.Cryptography/TASKS.md b/src/StellaOps.Cryptography/TASKS.md new file mode 100644 index 00000000..b334d159 --- /dev/null +++ b/src/StellaOps.Cryptography/TASKS.md @@ -0,0 +1,25 @@ +# Team 8 — Security Guild Task Board (UTC 2025-10-10) + +| ID | Status | Owner | Description | Dependencies | Exit Criteria | +|----|--------|-------|-------------|--------------|---------------| +| SEC1.A | TODO | Security Guild | Introduce `Argon2idPasswordHasher` backed by Konscious defaults. Wire options into `StandardPluginOptions` (`PasswordHashOptions`) and `StellaOpsAuthorityOptions.Security.PasswordHashing`. | PLG3, CORE3 | ✅ Hashes emit PHC string `$argon2id$v=19$m=19456,t=2,p=1$...`; ✅ `NeedsRehash` promotes PBKDF2 → Argon2; ✅ Integration tests cover tamper, legacy rehash, perf p95 < 250 ms. | +| SEC1.B | TODO | Security Guild | Add compile-time switch to enable libsodium/Core variants later (`STELLAOPS_CRYPTO_SODIUM`). Document build variable. | SEC1.A | ✅ Conditional compilation path compiles; ✅ README snippet in `docs/security/password-hashing.md`. | +| SEC2.A | TODO | Security Guild + Core | Define audit event contract (`AuthEventRecord`) with subject/client/scope/IP/outcome/correlationId and PII tags. | CORE5–CORE7 | ✅ Contract shipped in `StellaOps.Cryptography` (or shared abstractions); ✅ Docs in `docs/security/audit-events.md`. | +| SEC2.B | TODO | Security Guild | Emit audit records from OpenIddict handlers (password + client creds) and bootstrap APIs. Persist via `IAuthorityLoginAttemptStore`. | SEC2.A | ✅ Tests assert three flows (success/failure/lockout); ✅ Serilog output contains correlationId + PII tagging; ✅ Mongo store holds summary rows. | +| SEC3.A | BLOCKED (CORE8) | Security Guild + Core | Configure ASP.NET rate limiter (`AddRateLimiter`) with fixed-window policy keyed by IP + `client_id`. Apply to `/token` and `/internal/*`. | CORE8 completion | ✅ Middleware active; ✅ Configurable limits via options; ✅ Integration test hits 429. | +| SEC3.B | TODO | Security Guild | Document lockout + rate-limit tuning guidance and escalation thresholds. | SEC3.A | ✅ Section in `docs/security/rate-limits.md`; ✅ Includes SOC alert recommendations. | +| SEC4.A | TODO | Security Guild + DevOps | Define revocation JSON schema (`revocation_bundle.schema.json`) and detached JWS workflow. | CORE9, OPS3 | ✅ Schema + sample committed; ✅ CLI command `stellaops auth revoke export` scaffolded with acceptance tests; ✅ Verification script + docs. | +| SEC4.B | TODO | Security Guild | Integrate signing keys with crypto provider abstraction (initially ES256 via BCL). | SEC4.A, D5 | ✅ `ICryptoProvider.GetSigner` stub + default BCL signer; ✅ Unit tests verifying signature roundtrip. | +| SEC5.A | TODO | Security Guild | Author STRIDE threat model (`docs/security/authority-threat-model.md`) covering token, bootstrap, revocation, CLI, plugin surfaces. | All SEC1–SEC4 in progress | ✅ DFDs + trust boundaries drawn; ✅ Risk table with owners/actions; ✅ Follow-up backlog issues created. | +| D5.A | TODO | Security Guild | Flesh out `StellaOps.Cryptography` provider registry, policy, and DI helpers enabling sovereign crypto selection. | SEC1.A, SEC4.B | ✅ `ICryptoProviderRegistry` implementation with provider selection rules; ✅ `StellaOps.Cryptography.DependencyInjection` extensions; ✅ Tests covering fallback ordering. | + +## Notes +- Target Argon2 parameters follow OWASP Cheat Sheet (memory ≈ 19 MiB, iterations 2, parallelism 1). Allow overrides via configuration. +- When CORE8 lands, pair with Team 2 to expose request context information required by the rate limiter (client_id enrichment). +- Revocation bundle must be consumable offline; include issue timestamp, signing key metadata, and reasons. +- All crypto usage in Authority code should funnel through the new abstractions (`ICryptoProvider`), enabling future CryptoPro/OpenSSL providers. + +## Done Definition +- Code merges include unit/integration tests and documentation updates. +- `TASKS.md` status transitions (TODO → DOING → DONE/BLOCKED) must happen in the same PR as the work. +- Prior to marking DONE: run `dotnet test` for touched solutions and attach excerpt to PR description. diff --git a/src/StellaOps.DependencyInjection/IDependencyInjectionRoutine.cs b/src/StellaOps.DependencyInjection/IDependencyInjectionRoutine.cs new file mode 100644 index 00000000..b2d083f7 --- /dev/null +++ b/src/StellaOps.DependencyInjection/IDependencyInjectionRoutine.cs @@ -0,0 +1,11 @@ +using Microsoft.Extensions.Configuration; +using Microsoft.Extensions.DependencyInjection; + +namespace StellaOps.DependencyInjection; + +public interface IDependencyInjectionRoutine +{ + IServiceCollection Register( + IServiceCollection services, + IConfiguration configuration); +} \ No newline at end of file diff --git a/src/StellaOps.DependencyInjection/StellaOps.DependencyInjection.csproj b/src/StellaOps.DependencyInjection/StellaOps.DependencyInjection.csproj new file mode 100644 index 00000000..2be35349 --- /dev/null +++ b/src/StellaOps.DependencyInjection/StellaOps.DependencyInjection.csproj @@ -0,0 +1,14 @@ + + + + net10.0 + enable + enable + + + + + + + + \ No newline at end of file diff --git a/src/StellaOps.Feedser.Core.Tests/CanonicalMergerTests.cs b/src/StellaOps.Feedser.Core.Tests/CanonicalMergerTests.cs new file mode 100644 index 00000000..ae53ec7c --- /dev/null +++ b/src/StellaOps.Feedser.Core.Tests/CanonicalMergerTests.cs @@ -0,0 +1,239 @@ +using StellaOps.Feedser.Models; + +namespace StellaOps.Feedser.Core.Tests; + +public sealed class CanonicalMergerTests +{ + private static readonly DateTimeOffset BaseTimestamp = new(2025, 10, 10, 0, 0, 0, TimeSpan.Zero); + + [Fact] + public void Merge_PrefersGhsaTitleAndSummaryByPrecedence() + { + var merger = new CanonicalMerger(new FixedTimeProvider(BaseTimestamp.AddHours(6))); + + var ghsa = CreateAdvisory( + source: "ghsa", + advisoryKey: "GHSA-aaaa-bbbb-cccc", + title: "GHSA Title", + summary: "GHSA Summary", + modified: BaseTimestamp.AddHours(1)); + + var nvd = CreateAdvisory( + source: "nvd", + advisoryKey: "CVE-2025-0001", + title: "NVD Title", + summary: "NVD Summary", + modified: BaseTimestamp); + + var result = merger.Merge("CVE-2025-0001", ghsa, nvd, null); + + Assert.Equal("GHSA Title", result.Advisory.Title); + Assert.Equal("GHSA Summary", result.Advisory.Summary); + + Assert.Contains(result.Decisions, decision => + decision.Field == "summary" && + string.Equals(decision.SelectedSource, "ghsa", StringComparison.OrdinalIgnoreCase) && + string.Equals(decision.DecisionReason, "precedence", StringComparison.OrdinalIgnoreCase)); + + Assert.Contains(result.Advisory.Provenance, provenance => + string.Equals(provenance.Source, "ghsa", StringComparison.OrdinalIgnoreCase) && + string.Equals(provenance.Kind, "merge", StringComparison.OrdinalIgnoreCase) && + string.Equals(provenance.Value, "summary", StringComparison.OrdinalIgnoreCase) && + string.Equals(provenance.DecisionReason, "precedence", StringComparison.OrdinalIgnoreCase)); + } + + [Fact] + public void Merge_FreshnessOverrideUsesOsvSummaryWhenNewerByThreshold() + { + var merger = new CanonicalMerger(new FixedTimeProvider(BaseTimestamp.AddHours(10))); + + var ghsa = CreateAdvisory( + source: "ghsa", + advisoryKey: "GHSA-xxxx-yyyy-zzzz", + title: "Container Escape Vulnerability", + summary: "Initial GHSA summary.", + modified: BaseTimestamp); + + var osv = CreateAdvisory( + source: "osv", + advisoryKey: "GHSA-xxxx-yyyy-zzzz", + title: "Container Escape Vulnerability", + summary: "OSV summary with additional mitigation steps.", + modified: BaseTimestamp.AddHours(72)); + + var result = merger.Merge("CVE-2025-9000", ghsa, null, osv); + + Assert.Equal("OSV summary with additional mitigation steps.", result.Advisory.Summary); + + Assert.Contains(result.Decisions, decision => + decision.Field == "summary" && + string.Equals(decision.SelectedSource, "osv", StringComparison.OrdinalIgnoreCase) && + string.Equals(decision.DecisionReason, "freshness_override", StringComparison.OrdinalIgnoreCase)); + + Assert.Contains(result.Advisory.Provenance, provenance => + string.Equals(provenance.Source, "osv", StringComparison.OrdinalIgnoreCase) && + string.Equals(provenance.Kind, "merge", StringComparison.OrdinalIgnoreCase) && + string.Equals(provenance.Value, "summary", StringComparison.OrdinalIgnoreCase) && + string.Equals(provenance.DecisionReason, "freshness_override", StringComparison.OrdinalIgnoreCase)); + } + + [Fact] + public void Merge_AffectedPackagesPreferOsvPrecedence() + { + var merger = new CanonicalMerger(new FixedTimeProvider(BaseTimestamp.AddHours(4))); + + var ghsaPackage = new AffectedPackage( + AffectedPackageTypes.SemVer, + "pkg:npm/example@1", + platform: null, + versionRanges: new[] + { + new AffectedVersionRange( + rangeKind: "semver", + introducedVersion: null, + fixedVersion: "1.2.3", + lastAffectedVersion: null, + rangeExpression: "<1.2.3", + provenance: CreateProvenance("ghsa", ProvenanceFieldMasks.VersionRanges), + primitives: null) + }, + statuses: new[] + { + new AffectedPackageStatus( + "affected", + CreateProvenance("ghsa", ProvenanceFieldMasks.PackageStatuses)) + }, + provenance: new[] { CreateProvenance("ghsa", ProvenanceFieldMasks.AffectedPackages) }, + normalizedVersions: Array.Empty()); + + var nvdPackage = new AffectedPackage( + AffectedPackageTypes.SemVer, + "pkg:npm/example@1", + platform: null, + versionRanges: new[] + { + new AffectedVersionRange( + rangeKind: "semver", + introducedVersion: null, + fixedVersion: "1.2.4", + lastAffectedVersion: null, + rangeExpression: "<1.2.4", + provenance: CreateProvenance("nvd", ProvenanceFieldMasks.VersionRanges), + primitives: null) + }, + statuses: Array.Empty(), + provenance: new[] { CreateProvenance("nvd", ProvenanceFieldMasks.AffectedPackages) }, + normalizedVersions: Array.Empty()); + + var osvPackage = new AffectedPackage( + AffectedPackageTypes.SemVer, + "pkg:npm/example@1", + platform: null, + versionRanges: new[] + { + new AffectedVersionRange( + rangeKind: "semver", + introducedVersion: "1.0.0", + fixedVersion: "1.2.5", + lastAffectedVersion: null, + rangeExpression: ">=1.0.0,<1.2.5", + provenance: CreateProvenance("osv", ProvenanceFieldMasks.VersionRanges), + primitives: null) + }, + statuses: Array.Empty(), + provenance: new[] { CreateProvenance("osv", ProvenanceFieldMasks.AffectedPackages) }, + normalizedVersions: Array.Empty()); + + var ghsa = CreateAdvisory("ghsa", "GHSA-1234", "GHSA Title", null, BaseTimestamp.AddHours(1), packages: new[] { ghsaPackage }); + var nvd = CreateAdvisory("nvd", "CVE-2025-1111", "NVD Title", null, BaseTimestamp.AddHours(2), packages: new[] { nvdPackage }); + var osv = CreateAdvisory("osv", "OSV-2025-xyz", "OSV Title", null, BaseTimestamp.AddHours(3), packages: new[] { osvPackage }); + + var result = merger.Merge("CVE-2025-1111", ghsa, nvd, osv); + + var package = Assert.Single(result.Advisory.AffectedPackages); + Assert.Equal("pkg:npm/example@1", package.Identifier); + Assert.Contains(package.Provenance, provenance => + string.Equals(provenance.Source, "osv", StringComparison.OrdinalIgnoreCase) && + string.Equals(provenance.Kind, "merge", StringComparison.OrdinalIgnoreCase) && + string.Equals(provenance.DecisionReason, "precedence", StringComparison.OrdinalIgnoreCase)); + + Assert.Contains(result.Decisions, decision => + decision.Field.StartsWith("affectedPackages", StringComparison.OrdinalIgnoreCase) && + string.Equals(decision.SelectedSource, "osv", StringComparison.OrdinalIgnoreCase) && + string.Equals(decision.DecisionReason, "precedence", StringComparison.OrdinalIgnoreCase)); + } + + [Fact] + public void Merge_CvssMetricsOrderedByPrecedence() + { + var merger = new CanonicalMerger(new FixedTimeProvider(BaseTimestamp.AddHours(5))); + + var nvdMetric = new CvssMetric("3.1", "CVSS:3.1/AV:N/AC:L/PR:N/UI:N/S:C/C:H/I:H/A:H", 9.8, "critical", CreateProvenance("nvd", ProvenanceFieldMasks.CvssMetrics)); + var ghsaMetric = new CvssMetric("3.0", "CVSS:3.0/AV:L/AC:L/PR:L/UI:R/S:U/C:H/I:H/A:H", 7.5, "high", CreateProvenance("ghsa", ProvenanceFieldMasks.CvssMetrics)); + + var nvd = CreateAdvisory("nvd", "CVE-2025-2000", "NVD Title", severity: null, modified: BaseTimestamp, metrics: new[] { nvdMetric }); + var ghsa = CreateAdvisory("ghsa", "GHSA-9999", "GHSA Title", severity: null, modified: BaseTimestamp.AddHours(1), metrics: new[] { ghsaMetric }); + + var result = merger.Merge("CVE-2025-2000", ghsa, nvd, null); + + Assert.Equal(2, result.Advisory.CvssMetrics.Length); + Assert.Equal("nvd", result.Decisions.Single(decision => decision.Field == "cvssMetrics").SelectedSource); + Assert.Equal("critical", result.Advisory.Severity); + Assert.Contains(result.Advisory.CvssMetrics, metric => metric.Vector == "CVSS:3.1/AV:N/AC:L/PR:N/UI:N/S:C/C:H/I:H/A:H"); + Assert.Contains(result.Advisory.CvssMetrics, metric => metric.Vector == "CVSS:3.0/AV:L/AC:L/PR:L/UI:R/S:U/C:H/I:H/A:H"); + } + + private static Advisory CreateAdvisory( + string source, + string advisoryKey, + string title, + string? summary = null, + DateTimeOffset? modified = null, + string? severity = null, + IEnumerable? packages = null, + IEnumerable? metrics = null) + { + var provenance = new AdvisoryProvenance( + source, + kind: "map", + value: advisoryKey, + recordedAt: modified ?? BaseTimestamp, + fieldMask: new[] { ProvenanceFieldMasks.Advisory }); + + return new Advisory( + advisoryKey, + title, + summary, + language: "en", + published: modified, + modified: modified, + severity: severity, + exploitKnown: false, + aliases: new[] { advisoryKey }, + credits: Array.Empty(), + references: Array.Empty(), + affectedPackages: packages ?? Array.Empty(), + cvssMetrics: metrics ?? Array.Empty(), + provenance: new[] { provenance }); + } + + private static AdvisoryProvenance CreateProvenance(string source, string fieldMask) + => new( + source, + kind: "map", + value: source, + recordedAt: BaseTimestamp, + fieldMask: new[] { fieldMask }); + + private sealed class FixedTimeProvider : TimeProvider + { + private readonly DateTimeOffset _utcNow; + + public FixedTimeProvider(DateTimeOffset utcNow) + { + _utcNow = utcNow; + } + + public override DateTimeOffset GetUtcNow() => _utcNow; + } +} diff --git a/src/StellaOps.Feedser.Core.Tests/JobCoordinatorTests.cs b/src/StellaOps.Feedser.Core.Tests/JobCoordinatorTests.cs new file mode 100644 index 00000000..5dcdcfbc --- /dev/null +++ b/src/StellaOps.Feedser.Core.Tests/JobCoordinatorTests.cs @@ -0,0 +1,483 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Logging.Abstractions; +using Microsoft.Extensions.Options; +using StellaOps.Feedser.Core.Jobs; + +namespace StellaOps.Feedser.Core.Tests; + +public sealed class JobCoordinatorTests +{ + [Fact] + public async Task TriggerAsync_RunCompletesSuccessfully() + { + var services = new ServiceCollection(); + services.AddTransient(); + services.AddLogging(); + using var provider = services.BuildServiceProvider(); + + var jobStore = new InMemoryJobStore(); + var leaseStore = new InMemoryLeaseStore(); + var jobOptions = new JobSchedulerOptions + { + DefaultLeaseDuration = TimeSpan.FromSeconds(5), + DefaultTimeout = TimeSpan.FromSeconds(10), + }; + + var definition = new JobDefinition( + Kind: "test:run", + JobType: typeof(SuccessfulJob), + Timeout: TimeSpan.FromSeconds(5), + LeaseDuration: TimeSpan.FromSeconds(2), + CronExpression: null, + Enabled: true); + jobOptions.Definitions.Add(definition.Kind, definition); + + using var diagnostics = new JobDiagnostics(); + var coordinator = new JobCoordinator( + Options.Create(jobOptions), + jobStore, + leaseStore, + provider.GetRequiredService(), + NullLogger.Instance, + NullLoggerFactory.Instance, + new TestTimeProvider(), + diagnostics); + + var result = await coordinator.TriggerAsync(definition.Kind, new Dictionary { ["foo"] = "bar" }, "unit-test", CancellationToken.None); + + Assert.Equal(JobTriggerOutcome.Accepted, result.Outcome); + var completed = await jobStore.Completion.Task.WaitAsync(TimeSpan.FromSeconds(2)); + Assert.Equal(JobRunStatus.Succeeded, completed.Status); + await leaseStore.WaitForReleaseAsync(TimeSpan.FromSeconds(1)); + Assert.True(leaseStore.ReleaseCount > 0); + Assert.Equal("bar", completed.Parameters["foo"]); + } + + [Fact] + public async Task TriggerAsync_MarksRunFailed_WhenLeaseReleaseFails() + { + var services = new ServiceCollection(); + services.AddTransient(); + services.AddLogging(); + using var provider = services.BuildServiceProvider(); + + var jobStore = new InMemoryJobStore(); + var leaseStore = new FailingLeaseStore + { + ThrowOnRelease = true, + }; + + var jobOptions = new JobSchedulerOptions + { + DefaultLeaseDuration = TimeSpan.FromSeconds(5), + DefaultTimeout = TimeSpan.FromSeconds(10), + }; + + var definition = new JobDefinition( + Kind: "test:run", + JobType: typeof(SuccessfulJob), + Timeout: TimeSpan.FromSeconds(5), + LeaseDuration: TimeSpan.FromSeconds(2), + CronExpression: null, + Enabled: true); + jobOptions.Definitions.Add(definition.Kind, definition); + + using var diagnostics = new JobDiagnostics(); + var coordinator = new JobCoordinator( + Options.Create(jobOptions), + jobStore, + leaseStore, + provider.GetRequiredService(), + NullLogger.Instance, + NullLoggerFactory.Instance, + new TestTimeProvider(), + diagnostics); + + var result = await coordinator.TriggerAsync(definition.Kind, null, "unit-test", CancellationToken.None); + + Assert.Equal(JobTriggerOutcome.Accepted, result.Outcome); + var completed = await jobStore.Completion.Task.WaitAsync(TimeSpan.FromSeconds(2)); + Assert.Equal(JobRunStatus.Failed, completed.Status); + Assert.NotNull(completed.Error); + Assert.Contains("Failed to release lease", completed.Error!, StringComparison.OrdinalIgnoreCase); + Assert.True(leaseStore.ReleaseAttempts > 0); + } + + [Fact] + public async Task TriggerAsync_MarksRunFailed_WhenLeaseHeartbeatFails() + { + var services = new ServiceCollection(); + services.AddTransient(); + services.AddLogging(); + using var provider = services.BuildServiceProvider(); + + var jobStore = new InMemoryJobStore(); + var leaseStore = new FailingLeaseStore + { + ThrowOnHeartbeat = true, + }; + + var jobOptions = new JobSchedulerOptions + { + DefaultLeaseDuration = TimeSpan.FromSeconds(2), + DefaultTimeout = TimeSpan.FromSeconds(10), + }; + + var definition = new JobDefinition( + Kind: "test:heartbeat", + JobType: typeof(SlowJob), + Timeout: TimeSpan.FromSeconds(5), + LeaseDuration: TimeSpan.FromSeconds(2), + CronExpression: null, + Enabled: true); + jobOptions.Definitions.Add(definition.Kind, definition); + + using var diagnostics = new JobDiagnostics(); + var coordinator = new JobCoordinator( + Options.Create(jobOptions), + jobStore, + leaseStore, + provider.GetRequiredService(), + NullLogger.Instance, + NullLoggerFactory.Instance, + new TestTimeProvider(), + diagnostics); + + var result = await coordinator.TriggerAsync(definition.Kind, null, "unit-test", CancellationToken.None); + + Assert.Equal(JobTriggerOutcome.Accepted, result.Outcome); + var completed = await jobStore.Completion.Task.WaitAsync(TimeSpan.FromSeconds(6)); + Assert.Equal(JobRunStatus.Failed, completed.Status); + Assert.NotNull(completed.Error); + Assert.Contains("Failed to heartbeat lease", completed.Error!, StringComparison.OrdinalIgnoreCase); + Assert.True(leaseStore.HeartbeatCount > 0); + } + + [Fact] + public async Task TriggerAsync_ReturnsAlreadyRunning_WhenLeaseUnavailable() + { + var services = new ServiceCollection(); + services.AddTransient(); + using var provider = services.BuildServiceProvider(); + + var jobStore = new InMemoryJobStore(); + var leaseStore = new InMemoryLeaseStore + { + NextLease = null, + }; + var jobOptions = new JobSchedulerOptions(); + var definition = new JobDefinition( + "test:run", + typeof(SuccessfulJob), + TimeSpan.FromSeconds(5), + TimeSpan.FromSeconds(2), + null, + true); + jobOptions.Definitions.Add(definition.Kind, definition); + + using var diagnostics = new JobDiagnostics(); + var coordinator = new JobCoordinator( + Options.Create(jobOptions), + jobStore, + leaseStore, + provider.GetRequiredService(), + NullLogger.Instance, + NullLoggerFactory.Instance, + new TestTimeProvider(), + diagnostics); + + var result = await coordinator.TriggerAsync(definition.Kind, null, "unit-test", CancellationToken.None); + + Assert.Equal(JobTriggerOutcome.AlreadyRunning, result.Outcome); + Assert.False(jobStore.CreatedRuns.Any()); + } + + [Fact] + public async Task TriggerAsync_ReturnsInvalidParameters_ForUnsupportedPayload() + { + var services = new ServiceCollection(); + services.AddTransient(); + using var provider = services.BuildServiceProvider(); + + var jobStore = new InMemoryJobStore(); + var leaseStore = new InMemoryLeaseStore(); + var jobOptions = new JobSchedulerOptions(); + var definition = new JobDefinition( + "test:run", + typeof(SuccessfulJob), + TimeSpan.FromSeconds(5), + TimeSpan.FromSeconds(2), + null, + true); + jobOptions.Definitions.Add(definition.Kind, definition); + + using var diagnostics = new JobDiagnostics(); + var coordinator = new JobCoordinator( + Options.Create(jobOptions), + jobStore, + leaseStore, + provider.GetRequiredService(), + NullLogger.Instance, + NullLoggerFactory.Instance, + new TestTimeProvider(), + diagnostics); + + var parameters = new Dictionary + { + ["bad"] = new object(), + }; + + var result = await coordinator.TriggerAsync(definition.Kind, parameters, "unit-test", CancellationToken.None); + + Assert.Equal(JobTriggerOutcome.InvalidParameters, result.Outcome); + Assert.Contains("unsupported", result.ErrorMessage, StringComparison.OrdinalIgnoreCase); + Assert.False(jobStore.CreatedRuns.Any()); + } + + [Fact] + public async Task TriggerAsync_CancelsJobOnTimeout() + { + var services = new ServiceCollection(); + services.AddTransient(); + using var provider = services.BuildServiceProvider(); + + var jobStore = new InMemoryJobStore(); + var leaseStore = new InMemoryLeaseStore(); + var jobOptions = new JobSchedulerOptions + { + DefaultLeaseDuration = TimeSpan.FromSeconds(5), + DefaultTimeout = TimeSpan.FromMilliseconds(100), + }; + + var definition = new JobDefinition( + Kind: "test:timeout", + JobType: typeof(TimeoutJob), + Timeout: TimeSpan.FromMilliseconds(100), + LeaseDuration: TimeSpan.FromSeconds(2), + CronExpression: null, + Enabled: true); + jobOptions.Definitions.Add(definition.Kind, definition); + + using var diagnostics = new JobDiagnostics(); + var coordinator = new JobCoordinator( + Options.Create(jobOptions), + jobStore, + leaseStore, + provider.GetRequiredService(), + NullLogger.Instance, + NullLoggerFactory.Instance, + new TestTimeProvider(), + diagnostics); + + var result = await coordinator.TriggerAsync(definition.Kind, null, "unit-test", CancellationToken.None); + Assert.Equal(JobTriggerOutcome.Accepted, result.Outcome); + + var completed = await jobStore.Completion.Task.WaitAsync(TimeSpan.FromSeconds(2)); + Assert.Equal(JobRunStatus.Cancelled, completed.Status); + await leaseStore.WaitForReleaseAsync(TimeSpan.FromSeconds(1)); + Assert.True(leaseStore.ReleaseCount > 0); + } + + private sealed class SuccessfulJob : IJob + { + public Task ExecuteAsync(JobExecutionContext context, CancellationToken cancellationToken) + { + return Task.CompletedTask; + } + } + + private sealed class TimeoutJob : IJob + { + public async Task ExecuteAsync(JobExecutionContext context, CancellationToken cancellationToken) + { + await Task.Delay(TimeSpan.FromSeconds(5), cancellationToken); + } + } + + private sealed class SlowJob : IJob + { + public async Task ExecuteAsync(JobExecutionContext context, CancellationToken cancellationToken) + { + await Task.Delay(TimeSpan.FromSeconds(2), cancellationToken); + } + } + + private sealed class InMemoryJobStore : IJobStore + { + private readonly Dictionary _runs = new(); + public TaskCompletionSource Completion { get; } = new(TaskCreationOptions.RunContinuationsAsynchronously); + public List CreatedRuns { get; } = new(); + + public Task CreateAsync(JobRunCreateRequest request, CancellationToken cancellationToken) + { + var run = new JobRunSnapshot( + Guid.NewGuid(), + request.Kind, + JobRunStatus.Pending, + request.CreatedAt, + null, + null, + request.Trigger, + request.ParametersHash, + null, + request.Timeout, + request.LeaseDuration, + request.Parameters); + _runs[run.RunId] = run; + CreatedRuns.Add(run); + return Task.FromResult(run); + } + + public Task TryStartAsync(Guid runId, DateTimeOffset startedAt, CancellationToken cancellationToken) + { + if (_runs.TryGetValue(runId, out var run)) + { + var updated = run with { Status = JobRunStatus.Running, StartedAt = startedAt }; + _runs[runId] = updated; + return Task.FromResult(updated); + } + + return Task.FromResult(null); + } + + public Task TryCompleteAsync(Guid runId, JobRunCompletion completion, CancellationToken cancellationToken) + { + if (_runs.TryGetValue(runId, out var run)) + { + var updated = run with { Status = completion.Status, CompletedAt = completion.CompletedAt, Error = completion.Error }; + _runs[runId] = updated; + Completion.TrySetResult(updated); + return Task.FromResult(updated); + } + + return Task.FromResult(null); + } + + public Task FindAsync(Guid runId, CancellationToken cancellationToken) + { + _runs.TryGetValue(runId, out var run); + return Task.FromResult(run); + } + + public Task> GetRecentRunsAsync(string? kind, int limit, CancellationToken cancellationToken) + { + var query = _runs.Values.AsEnumerable(); + if (!string.IsNullOrWhiteSpace(kind)) + { + query = query.Where(r => r.Kind == kind); + } + + return Task.FromResult>(query.OrderByDescending(r => r.CreatedAt).Take(limit).ToArray()); + } + + public Task> GetActiveRunsAsync(CancellationToken cancellationToken) + { + return Task.FromResult>(_runs.Values.Where(r => r.Status is JobRunStatus.Pending or JobRunStatus.Running).ToArray()); + } + + public Task GetLastRunAsync(string kind, CancellationToken cancellationToken) + { + var run = _runs.Values + .Where(r => r.Kind == kind) + .OrderByDescending(r => r.CreatedAt) + .FirstOrDefault(); + return Task.FromResult(run); + } + + public Task> GetLastRunsAsync(IEnumerable kinds, CancellationToken cancellationToken) + { + var results = new Dictionary(StringComparer.Ordinal); + foreach (var kind in kinds.Distinct(StringComparer.Ordinal)) + { + var run = _runs.Values + .Where(r => r.Kind == kind) + .OrderByDescending(r => r.CreatedAt) + .FirstOrDefault(); + if (run is not null) + { + results[kind] = run; + } + } + + return Task.FromResult>(results); + } + } + + private sealed class InMemoryLeaseStore : ILeaseStore + { + public JobLease? NextLease { get; set; } = new JobLease("job:test:run", "holder", DateTimeOffset.UtcNow, DateTimeOffset.UtcNow, TimeSpan.FromSeconds(2), DateTimeOffset.UtcNow.AddSeconds(2)); + public int HeartbeatCount { get; private set; } + public int ReleaseCount { get; private set; } + private readonly TaskCompletionSource _released = new(TaskCreationOptions.RunContinuationsAsynchronously); + + public Task TryAcquireAsync(string key, string holder, TimeSpan leaseDuration, DateTimeOffset now, CancellationToken cancellationToken) + { + return Task.FromResult(NextLease); + } + + public Task HeartbeatAsync(string key, string holder, TimeSpan leaseDuration, DateTimeOffset now, CancellationToken cancellationToken) + { + HeartbeatCount++; + NextLease = new JobLease(key, holder, now, now, leaseDuration, now.Add(leaseDuration)); + return Task.FromResult(NextLease); + } + + public Task ReleaseAsync(string key, string holder, CancellationToken cancellationToken) + { + ReleaseCount++; + _released.TrySetResult(true); + return Task.FromResult(true); + } + + public Task WaitForReleaseAsync(TimeSpan timeout) + => _released.Task.WaitAsync(timeout); + } + + private sealed class FailingLeaseStore : ILeaseStore + { + public bool ThrowOnHeartbeat { get; set; } + public bool ThrowOnRelease { get; set; } + + public int HeartbeatCount { get; private set; } + public int ReleaseAttempts { get; private set; } + + public Task TryAcquireAsync(string key, string holder, TimeSpan leaseDuration, DateTimeOffset now, CancellationToken cancellationToken) + { + var lease = new JobLease(key, holder, now, now, leaseDuration, now.Add(leaseDuration)); + return Task.FromResult(lease); + } + + public Task HeartbeatAsync(string key, string holder, TimeSpan leaseDuration, DateTimeOffset now, CancellationToken cancellationToken) + { + HeartbeatCount++; + if (ThrowOnHeartbeat) + { + throw new InvalidOperationException("Lease heartbeat failed"); + } + + var lease = new JobLease(key, holder, now, now, leaseDuration, now.Add(leaseDuration)); + return Task.FromResult(lease); + } + + public Task ReleaseAsync(string key, string holder, CancellationToken cancellationToken) + { + ReleaseAttempts++; + if (ThrowOnRelease) + { + throw new InvalidOperationException("Failed to release lease"); + } + + return Task.FromResult(true); + } + } + + private sealed class TestTimeProvider : TimeProvider + { + private DateTimeOffset _now = DateTimeOffset.Parse("2024-01-01T00:00:00Z"); + + public override DateTimeOffset GetUtcNow() => _now = _now.AddMilliseconds(100); + } +} diff --git a/src/StellaOps.Feedser.Core.Tests/JobPluginRegistrationExtensionsTests.cs b/src/StellaOps.Feedser.Core.Tests/JobPluginRegistrationExtensionsTests.cs new file mode 100644 index 00000000..3946b8e4 --- /dev/null +++ b/src/StellaOps.Feedser.Core.Tests/JobPluginRegistrationExtensionsTests.cs @@ -0,0 +1,61 @@ +using System; +using System.Collections.Generic; +using System.IO; +using Microsoft.Extensions.Configuration; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Options; +using StellaOps.Feedser.Core.Jobs; +using StellaOps.Plugin.Hosting; + +namespace StellaOps.Feedser.Core.Tests; + +public sealed class JobPluginRegistrationExtensionsTests +{ + [Fact] + public void RegisterJobPluginRoutines_LoadsPluginsAndRegistersDefinitions() + { + var services = new ServiceCollection(); + services.AddJobScheduler(); + + var configuration = new ConfigurationBuilder() + .AddInMemoryCollection(new Dictionary + { + ["plugin:test:timeoutSeconds"] = "45", + }) + .Build(); + + var assemblyPath = typeof(JobPluginRegistrationExtensionsTests).Assembly.Location; + var pluginDirectory = Path.GetDirectoryName(assemblyPath)!; + var pluginFile = Path.GetFileName(assemblyPath); + + var options = new PluginHostOptions + { + BaseDirectory = pluginDirectory, + PluginsDirectory = pluginDirectory, + EnsureDirectoryExists = false, + RecursiveSearch = false, + }; + options.SearchPatterns.Add(pluginFile); + + services.RegisterJobPluginRoutines(configuration, options); + + Assert.Contains( + services, + descriptor => descriptor.ServiceType == typeof(PluginHostResult)); + + Assert.Contains( + services, + descriptor => descriptor.ServiceType.FullName == typeof(PluginRoutineExecuted).FullName); + + using var provider = services.BuildServiceProvider(); + var schedulerOptions = provider.GetRequiredService>().Value; + + Assert.True(schedulerOptions.Definitions.TryGetValue(PluginJob.JobKind, out var definition)); + Assert.NotNull(definition); + Assert.Equal(PluginJob.JobKind, definition.Kind); + Assert.Equal("StellaOps.Feedser.Core.Tests.PluginJob", definition.JobType.FullName); + Assert.Equal(TimeSpan.FromSeconds(45), definition.Timeout); + Assert.Equal(TimeSpan.FromSeconds(5), definition.LeaseDuration); + Assert.Equal("*/10 * * * *", definition.CronExpression); + } +} diff --git a/src/StellaOps.Feedser.Core.Tests/JobSchedulerBuilderTests.cs b/src/StellaOps.Feedser.Core.Tests/JobSchedulerBuilderTests.cs new file mode 100644 index 00000000..034546cf --- /dev/null +++ b/src/StellaOps.Feedser.Core.Tests/JobSchedulerBuilderTests.cs @@ -0,0 +1,70 @@ +using System; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Options; +using StellaOps.Feedser.Core.Jobs; + +namespace StellaOps.Feedser.Core.Tests; + +public sealed class JobSchedulerBuilderTests +{ + [Fact] + public void AddJob_RegistersDefinitionWithExplicitMetadata() + { + var services = new ServiceCollection(); + var builder = services.AddJobScheduler(); + + builder.AddJob( + kind: "jobs:test", + cronExpression: "*/5 * * * *", + timeout: TimeSpan.FromMinutes(42), + leaseDuration: TimeSpan.FromMinutes(7), + enabled: false); + + using var provider = services.BuildServiceProvider(); + var options = provider.GetRequiredService>().Value; + + Assert.True(options.Definitions.TryGetValue("jobs:test", out var definition)); + Assert.NotNull(definition); + Assert.Equal(typeof(TestJob), definition.JobType); + Assert.Equal(TimeSpan.FromMinutes(42), definition.Timeout); + Assert.Equal(TimeSpan.FromMinutes(7), definition.LeaseDuration); + Assert.Equal("*/5 * * * *", definition.CronExpression); + Assert.False(definition.Enabled); + } + + [Fact] + public void AddJob_UsesDefaults_WhenOptionalMetadataExcluded() + { + var services = new ServiceCollection(); + var builder = services.AddJobScheduler(options => + { + options.DefaultTimeout = TimeSpan.FromSeconds(123); + options.DefaultLeaseDuration = TimeSpan.FromSeconds(45); + }); + + builder.AddJob(kind: "jobs:defaults"); + + using var provider = services.BuildServiceProvider(); + var options = provider.GetRequiredService>().Value; + + Assert.True(options.Definitions.TryGetValue("jobs:defaults", out var definition)); + Assert.NotNull(definition); + Assert.Equal(typeof(DefaultedJob), definition.JobType); + Assert.Equal(TimeSpan.FromSeconds(123), definition.Timeout); + Assert.Equal(TimeSpan.FromSeconds(45), definition.LeaseDuration); + Assert.Null(definition.CronExpression); + Assert.True(definition.Enabled); + } + + private sealed class TestJob : IJob + { + public Task ExecuteAsync(JobExecutionContext context, CancellationToken cancellationToken) + => Task.CompletedTask; + } + + private sealed class DefaultedJob : IJob + { + public Task ExecuteAsync(JobExecutionContext context, CancellationToken cancellationToken) + => Task.CompletedTask; + } +} diff --git a/src/StellaOps.Feedser.Core.Tests/PluginRoutineFixtures.cs b/src/StellaOps.Feedser.Core.Tests/PluginRoutineFixtures.cs new file mode 100644 index 00000000..3ba6defa --- /dev/null +++ b/src/StellaOps.Feedser.Core.Tests/PluginRoutineFixtures.cs @@ -0,0 +1,42 @@ +using System; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.Extensions.Configuration; +using Microsoft.Extensions.DependencyInjection; +using StellaOps.DependencyInjection; +using StellaOps.Feedser.Core.Jobs; + +namespace StellaOps.Feedser.Core.Tests; + +public sealed class TestPluginRoutine : IDependencyInjectionRoutine +{ + public IServiceCollection Register(IServiceCollection services, IConfiguration configuration) + { + ArgumentNullException.ThrowIfNull(services); + ArgumentNullException.ThrowIfNull(configuration); + + var builder = new JobSchedulerBuilder(services); + var timeoutSeconds = configuration.GetValue("plugin:test:timeoutSeconds") ?? 30; + + builder.AddJob( + PluginJob.JobKind, + cronExpression: "*/10 * * * *", + timeout: TimeSpan.FromSeconds(timeoutSeconds), + leaseDuration: TimeSpan.FromSeconds(5)); + + services.AddSingleton(); + return services; + } +} + +public sealed class PluginRoutineExecuted +{ +} + +public sealed class PluginJob : IJob +{ + public const string JobKind = "plugin:test"; + + public Task ExecuteAsync(JobExecutionContext context, CancellationToken cancellationToken) + => Task.CompletedTask; +} diff --git a/src/StellaOps.Feedser.Core.Tests/StellaOps.Feedser.Core.Tests.csproj b/src/StellaOps.Feedser.Core.Tests/StellaOps.Feedser.Core.Tests.csproj new file mode 100644 index 00000000..9e7a8693 --- /dev/null +++ b/src/StellaOps.Feedser.Core.Tests/StellaOps.Feedser.Core.Tests.csproj @@ -0,0 +1,10 @@ + + + net10.0 + enable + enable + + + + + diff --git a/src/StellaOps.Feedser.Core/AGENTS.md b/src/StellaOps.Feedser.Core/AGENTS.md new file mode 100644 index 00000000..fc658bf7 --- /dev/null +++ b/src/StellaOps.Feedser.Core/AGENTS.md @@ -0,0 +1,32 @@ +# AGENTS +## Role +Job orchestration and lifecycle. Registers job definitions, schedules execution, triggers runs, reports status for connectors and exporters. +## Scope +- Contracts: IJob (execute with CancellationToken), JobRunStatus, JobTriggerOutcome/Result. +- Registration: JobSchedulerBuilder.AddJob(kind, cronExpression?, timeout?, leaseDuration?); options recorded in JobSchedulerOptions. +- Plugin host integration discovers IJob providers via registered IDependencyInjectionRoutine implementations. +- Coordination: start/stop, single-flight via storage locks/leases, run bookkeeping (status, timings, errors). +- Triggering: manual/cron/API; parameterized runs; idempotent rejection if already running. +- Surfacing: enumerate definitions, last run, recent runs, active runs to WebService endpoints. +## Participants +- WebService exposes REST endpoints for definitions, runs, active, and trigger. +- Storage.Mongo persists job definitions metadata, run documents, and leases (locks collection). +- Source connectors and Exporters implement IJob and are registered into the scheduler via DI and Plugin routines. +- Models/Merge/Export are invoked indirectly through jobs. +- Plugin host runtime loads dependency injection routines that register job definitions. +## Interfaces & contracts +- Kind naming: family:source:verb (e.g., nvd:fetch, redhat:map, export:trivy-db). +- Timeout and lease duration enforce cancellation and duplicate-prevention. +- TimeProvider used for deterministic timing in tests. +## In/Out of scope +In: job lifecycle, registration, trigger semantics, run metadata. +Out: business logic of connectors/exporters, HTTP handlers (owned by WebService). +## Observability & security expectations +- Metrics: job.run.started/succeeded/failed, job.durationMs, job.concurrent.rejected, job.alreadyRunning. +- Logs: kind, trigger, params hash, lease holder, outcome; redact params containing secrets. +- Honor CancellationToken early and often. +## Tests +- Author and review coverage in `../StellaOps.Feedser.Core.Tests`. +- Shared fixtures (e.g., `MongoIntegrationFixture`, `ConnectorTestHarness`) live in `../StellaOps.Feedser.Testing`. +- Keep fixtures deterministic; match new cases to real-world advisories or regression scenarios. + diff --git a/src/StellaOps.Feedser.Core/CanonicalMergeResult.cs b/src/StellaOps.Feedser.Core/CanonicalMergeResult.cs new file mode 100644 index 00000000..ad26a3fd --- /dev/null +++ b/src/StellaOps.Feedser.Core/CanonicalMergeResult.cs @@ -0,0 +1,19 @@ +using System.Collections.Immutable; +using StellaOps.Feedser.Models; + +namespace StellaOps.Feedser.Core; + +/// +/// Result emitted by describing the merged advisory and analytics about key decisions. +/// +public sealed record CanonicalMergeResult(Advisory Advisory, ImmutableArray Decisions); + +/// +/// Describes how a particular canonical field was chosen during conflict resolution. +/// +public sealed record FieldDecision( + string Field, + string? SelectedSource, + string DecisionReason, + DateTimeOffset? SelectedModified, + ImmutableArray ConsideredSources); diff --git a/src/StellaOps.Feedser.Core/CanonicalMerger.cs b/src/StellaOps.Feedser.Core/CanonicalMerger.cs new file mode 100644 index 00000000..0a47cedc --- /dev/null +++ b/src/StellaOps.Feedser.Core/CanonicalMerger.cs @@ -0,0 +1,669 @@ +using System.Collections.Immutable; +using System.Security.Cryptography; +using System.Text; +using StellaOps.Feedser.Models; + +namespace StellaOps.Feedser.Core; + +/// +/// Resolves conflicts between GHSA, NVD, and OSV advisories into a single canonical advisory following +/// DEDUP_CONFLICTS_RESOLUTION_ALGO.md. +/// +public sealed class CanonicalMerger +{ + private const string GhsaSource = "ghsa"; + private const string NvdSource = "nvd"; + private const string OsvSource = "osv"; + + private static readonly ImmutableDictionary FieldPrecedence = new Dictionary(StringComparer.OrdinalIgnoreCase) + { + ["title"] = new[] { GhsaSource, NvdSource, OsvSource }, + ["summary"] = new[] { GhsaSource, NvdSource, OsvSource }, + ["language"] = new[] { GhsaSource, NvdSource, OsvSource }, + ["severity"] = new[] { NvdSource, GhsaSource, OsvSource }, + ["references"] = new[] { GhsaSource, NvdSource, OsvSource }, + ["credits"] = new[] { GhsaSource, OsvSource, NvdSource }, + ["affectedPackages"] = new[] { OsvSource, GhsaSource, NvdSource }, + ["cvssMetrics"] = new[] { NvdSource, GhsaSource, OsvSource }, + }.ToImmutableDictionary(StringComparer.OrdinalIgnoreCase); + + private static readonly ImmutableHashSet FreshnessSensitiveFields = ImmutableHashSet.Create( + StringComparer.OrdinalIgnoreCase, + "title", + "summary", + "references", + "credits", + "affectedPackages"); + + private static readonly ImmutableDictionary SourceOrder = new Dictionary(StringComparer.OrdinalIgnoreCase) + { + [GhsaSource] = 0, + [NvdSource] = 1, + [OsvSource] = 2, + }.ToImmutableDictionary(StringComparer.OrdinalIgnoreCase); + + private readonly TimeProvider _timeProvider; + private readonly TimeSpan _freshnessThreshold; + + public CanonicalMerger(TimeProvider? timeProvider = null, TimeSpan? freshnessThreshold = null) + { + _timeProvider = timeProvider ?? TimeProvider.System; + _freshnessThreshold = freshnessThreshold ?? TimeSpan.FromHours(48); + } + + public CanonicalMergeResult Merge(string advisoryKey, Advisory? ghsa, Advisory? nvd, Advisory? osv) + { + ArgumentException.ThrowIfNullOrWhiteSpace(advisoryKey); + + var candidates = BuildCandidates(ghsa, nvd, osv); + if (candidates.Count == 0) + { + throw new ArgumentException("At least one advisory must be provided.", nameof(advisoryKey)); + } + + var now = _timeProvider.GetUtcNow(); + var decisions = new List(); + var provenanceSet = new HashSet(); + + foreach (var candidate in candidates) + { + foreach (var existingProvenance in candidate.Advisory.Provenance) + { + provenanceSet.Add(existingProvenance); + } + } + + var titleSelection = SelectStringField("title", candidates, advisory => advisory.Title, isFreshnessSensitive: true); + if (titleSelection.HasValue) + { + decisions.Add(titleSelection.Decision); + AddMergeProvenance(provenanceSet, titleSelection, now, ProvenanceFieldMasks.Advisory); + } + + var summarySelection = SelectStringField("summary", candidates, advisory => advisory.Summary, isFreshnessSensitive: true); + if (summarySelection.HasValue) + { + decisions.Add(summarySelection.Decision); + AddMergeProvenance(provenanceSet, summarySelection, now, ProvenanceFieldMasks.Advisory); + } + + var languageSelection = SelectStringField("language", candidates, advisory => advisory.Language, isFreshnessSensitive: false); + if (languageSelection.HasValue) + { + decisions.Add(languageSelection.Decision); + AddMergeProvenance(provenanceSet, languageSelection, now, ProvenanceFieldMasks.Advisory); + } + + var topLevelSeveritySelection = SelectStringField("severity", candidates, advisory => advisory.Severity, isFreshnessSensitive: false); + if (topLevelSeveritySelection.HasValue) + { + decisions.Add(topLevelSeveritySelection.Decision); + AddMergeProvenance(provenanceSet, topLevelSeveritySelection, now, ProvenanceFieldMasks.Advisory); + } + + var aliases = MergeAliases(candidates); + var creditsResult = MergeCredits(candidates); + if (creditsResult.Decision is not null) + { + decisions.Add(creditsResult.Decision); + } + + var referencesResult = MergeReferences(candidates); + if (referencesResult.Decision is not null) + { + decisions.Add(referencesResult.Decision); + } + + var packagesResult = MergePackages(candidates, now); + decisions.AddRange(packagesResult.Decisions); + foreach (var packageProvenance in packagesResult.AdditionalProvenance) + { + provenanceSet.Add(packageProvenance); + } + + var metricsResult = MergeCvssMetrics(candidates); + if (metricsResult.Decision is not null) + { + decisions.Add(metricsResult.Decision); + } + + var exploitKnown = candidates.Any(candidate => candidate.Advisory.ExploitKnown); + var published = candidates + .Select(candidate => candidate.Advisory.Published) + .Where(static value => value.HasValue) + .Select(static value => value!.Value) + .DefaultIfEmpty() + .Min(); + var modified = candidates + .Select(candidate => candidate.Advisory.Modified) + .Where(static value => value.HasValue) + .Select(static value => value!.Value) + .DefaultIfEmpty() + .Max(); + + var title = titleSelection.Value ?? ghsa?.Title ?? nvd?.Title ?? osv?.Title ?? advisoryKey; + var summary = summarySelection.Value ?? ghsa?.Summary ?? nvd?.Summary ?? osv?.Summary; + var language = languageSelection.Value ?? ghsa?.Language ?? nvd?.Language ?? osv?.Language; + var severity = topLevelSeveritySelection.Value ?? metricsResult.CanonicalSeverity ?? ghsa?.Severity ?? nvd?.Severity ?? osv?.Severity; + + if (string.IsNullOrWhiteSpace(title)) + { + title = advisoryKey; + } + + var provenance = provenanceSet + .OrderBy(static p => p.Source, StringComparer.Ordinal) + .ThenBy(static p => p.Kind, StringComparer.Ordinal) + .ThenBy(static p => p.RecordedAt) + .ToImmutableArray(); + + var advisory = new Advisory( + advisoryKey, + title, + summary, + language, + published == DateTimeOffset.MinValue ? null : published, + modified == DateTimeOffset.MinValue ? null : modified, + severity, + exploitKnown, + aliases, + creditsResult.Credits, + referencesResult.References, + packagesResult.Packages, + metricsResult.Metrics, + provenance); + + return new CanonicalMergeResult( + advisory, + decisions + .OrderBy(static d => d.Field, StringComparer.Ordinal) + .ThenBy(static d => d.SelectedSource, StringComparer.Ordinal) + .ToImmutableArray()); + } + + private ImmutableArray MergeAliases(List candidates) + { + var set = new HashSet(StringComparer.OrdinalIgnoreCase); + foreach (var candidate in candidates) + { + foreach (var alias in candidate.Advisory.Aliases) + { + if (!string.IsNullOrWhiteSpace(alias)) + { + set.Add(alias); + } + } + } + + return set.Count == 0 ? ImmutableArray.Empty : set.OrderBy(static value => value, StringComparer.Ordinal).ToImmutableArray(); + } + + private CreditsMergeResult MergeCredits(List candidates) + { + var precedence = GetPrecedence("credits"); + var map = new Dictionary(StringComparer.OrdinalIgnoreCase); + var considered = new HashSet(StringComparer.OrdinalIgnoreCase); + + foreach (var candidate in candidates) + { + foreach (var credit in candidate.Advisory.Credits) + { + var key = $"{credit.DisplayName}|{credit.Role}"; + considered.Add(candidate.Source); + + if (!map.TryGetValue(key, out var existing)) + { + map[key] = new CreditSelection(credit, candidate.Source, candidate.Modified); + continue; + } + + var candidateRank = GetRank(candidate.Source, precedence); + var existingRank = GetRank(existing.Source, precedence); + + if (candidateRank < existingRank || + (candidateRank == existingRank && candidate.Modified > existing.Modified)) + { + map[key] = new CreditSelection(credit, candidate.Source, candidate.Modified); + } + } + } + + var credits = map.Values.Select(static s => s.Credit).ToImmutableArray(); + FieldDecision? decision = null; + + if (considered.Count > 0) + { + decision = new FieldDecision( + Field: "credits", + SelectedSource: null, + DecisionReason: "union", + SelectedModified: null, + ConsideredSources: considered.OrderBy(static value => value, StringComparer.OrdinalIgnoreCase).ToImmutableArray()); + } + + return new CreditsMergeResult(credits, decision); + } + + private ReferencesMergeResult MergeReferences(List candidates) + { + var precedence = GetPrecedence("references"); + var map = new Dictionary(StringComparer.OrdinalIgnoreCase); + var considered = new HashSet(StringComparer.OrdinalIgnoreCase); + + foreach (var candidate in candidates) + { + foreach (var reference in candidate.Advisory.References) + { + if (string.IsNullOrWhiteSpace(reference.Url)) + { + continue; + } + + var key = reference.Url.Trim(); + considered.Add(candidate.Source); + + if (!map.TryGetValue(key, out var existing)) + { + map[key] = new ReferenceSelection(reference, candidate.Source, candidate.Modified); + continue; + } + + var candidateRank = GetRank(candidate.Source, precedence); + var existingRank = GetRank(existing.Source, precedence); + + if (candidateRank < existingRank || + (candidateRank == existingRank && candidate.Modified > existing.Modified)) + { + map[key] = new ReferenceSelection(reference, candidate.Source, candidate.Modified); + } + } + } + + var references = map.Values.Select(static s => s.Reference).ToImmutableArray(); + FieldDecision? decision = null; + + if (considered.Count > 0) + { + decision = new FieldDecision( + Field: "references", + SelectedSource: null, + DecisionReason: "union", + SelectedModified: null, + ConsideredSources: considered.OrderBy(static value => value, StringComparer.OrdinalIgnoreCase).ToImmutableArray()); + } + + return new ReferencesMergeResult(references, decision); + } + + private PackagesMergeResult MergePackages(List candidates, DateTimeOffset now) + { + var precedence = GetPrecedence("affectedPackages"); + var map = new Dictionary(StringComparer.OrdinalIgnoreCase); + var decisions = new List(); + var additionalProvenance = new List(); + + foreach (var candidate in candidates) + { + foreach (var package in candidate.Advisory.AffectedPackages) + { + var key = CreatePackageKey(package); + var consideredSources = new HashSet(StringComparer.OrdinalIgnoreCase) { candidate.Source }; + + if (!map.TryGetValue(key, out var existing)) + { + var enriched = AppendMergeProvenance(package, candidate.Source, "precedence", now); + additionalProvenance.Add(enriched.MergeProvenance); + map[key] = new PackageSelection(enriched.Package, candidate.Source, candidate.Modified); + + decisions.Add(new FieldDecision( + Field: $"affectedPackages[{key}]", + SelectedSource: candidate.Source, + DecisionReason: "precedence", + SelectedModified: candidate.Modified, + ConsideredSources: consideredSources.ToImmutableArray())); + continue; + } + + consideredSources.Add(existing.Source); + + var candidateRank = GetRank(candidate.Source, precedence); + var existingRank = GetRank(existing.Source, precedence); + var freshness = candidate.Modified - existing.Modified; + var reason = string.Empty; + var shouldReplace = false; + + if (candidateRank < existingRank) + { + shouldReplace = true; + reason = "precedence"; + } + else if (candidateRank > existingRank && freshness >= _freshnessThreshold) + { + shouldReplace = true; + reason = "freshness_override"; + } + else if (candidateRank == existingRank && candidate.Modified > existing.Modified) + { + shouldReplace = true; + reason = "tie_breaker"; + } + + if (!shouldReplace) + { + continue; + } + + var enrichedPackage = AppendMergeProvenance(package, candidate.Source, reason, now); + additionalProvenance.Add(enrichedPackage.MergeProvenance); + map[key] = new PackageSelection(enrichedPackage.Package, candidate.Source, candidate.Modified); + + decisions.Add(new FieldDecision( + Field: $"affectedPackages[{key}]", + SelectedSource: candidate.Source, + DecisionReason: reason, + SelectedModified: candidate.Modified, + ConsideredSources: consideredSources.ToImmutableArray())); + } + } + + var packages = map.Values.Select(static s => s.Package).ToImmutableArray(); + return new PackagesMergeResult(packages, decisions, additionalProvenance); + } + + private CvssMergeResult MergeCvssMetrics(List candidates) + { + var precedence = GetPrecedence("cvssMetrics"); + var map = new Dictionary(StringComparer.OrdinalIgnoreCase); + var considered = new HashSet(StringComparer.OrdinalIgnoreCase); + + foreach (var candidate in candidates) + { + foreach (var metric in candidate.Advisory.CvssMetrics) + { + var key = $"{metric.Version}|{metric.Vector}"; + considered.Add(candidate.Source); + + if (!map.TryGetValue(key, out var existing)) + { + map[key] = new MetricSelection(metric, candidate.Source, candidate.Modified); + continue; + } + + var candidateRank = GetRank(candidate.Source, precedence); + var existingRank = GetRank(existing.Source, precedence); + + if (candidateRank < existingRank || + (candidateRank == existingRank && candidate.Modified > existing.Modified)) + { + map[key] = new MetricSelection(metric, candidate.Source, candidate.Modified); + } + } + } + + var orderedMetrics = map + .Values + .OrderBy(selection => GetRank(selection.Source, precedence)) + .ThenByDescending(selection => selection.Modified) + .Select(static selection => selection.Metric) + .ToImmutableArray(); + + FieldDecision? decision = null; + + if (considered.Count > 0) + { + var canonical = orderedMetrics.FirstOrDefault(); + decision = new FieldDecision( + Field: "cvssMetrics", + SelectedSource: canonical is null ? null : map[$"{canonical.Version}|{canonical.Vector}"].Source, + DecisionReason: "precedence", + SelectedModified: canonical is null ? null : map[$"{canonical.Version}|{canonical.Vector}"].Modified, + ConsideredSources: considered.OrderBy(static value => value, StringComparer.OrdinalIgnoreCase).ToImmutableArray()); + } + + var severity = orderedMetrics.FirstOrDefault()?.BaseSeverity; + return new CvssMergeResult(orderedMetrics, severity, decision); + } + + private static string CreatePackageKey(AffectedPackage package) + => string.Join('|', package.Type ?? string.Empty, package.Identifier ?? string.Empty, package.Platform ?? string.Empty); + + private static (AffectedPackage Package, AdvisoryProvenance MergeProvenance) AppendMergeProvenance( + AffectedPackage package, + string source, + string decisionReason, + DateTimeOffset recordedAt) + { + var provenance = new AdvisoryProvenance( + source, + kind: "merge", + value: CreatePackageKey(package), + recordedAt: recordedAt, + fieldMask: new[] { ProvenanceFieldMasks.AffectedPackages }, + decisionReason: decisionReason); + + var provenanceList = package.Provenance.ToBuilder(); + provenanceList.Add(provenance); + + var packageWithProvenance = new AffectedPackage( + package.Type, + package.Identifier, + package.Platform, + package.VersionRanges, + package.Statuses, + provenanceList, + package.NormalizedVersions); + + return (packageWithProvenance, provenance); + } + + private FieldSelection SelectStringField( + string field, + List candidates, + Func selector, + bool isFreshnessSensitive) + { + var precedence = GetPrecedence(field); + var valueCandidates = new List(); + + foreach (var candidate in candidates) + { + var value = Validation.TrimToNull(selector(candidate.Advisory)); + if (!string.IsNullOrEmpty(value)) + { + valueCandidates.Add(new ValueCandidate(candidate, value)); + } + } + + if (valueCandidates.Count == 0) + { + return FieldSelection.Empty; + } + + var consideredSources = valueCandidates + .Select(vc => vc.Candidate.Source) + .Distinct(StringComparer.OrdinalIgnoreCase) + .OrderBy(static source => source, StringComparer.OrdinalIgnoreCase) + .ToImmutableArray(); + + var best = valueCandidates + .OrderBy(vc => GetRank(vc.Candidate.Source, precedence)) + .ThenByDescending(vc => vc.Candidate.Modified) + .First(); + + var decisionReason = "precedence"; + + if (isFreshnessSensitive) + { + var freshnessOverride = valueCandidates + .Where(vc => GetRank(vc.Candidate.Source, precedence) > GetRank(best.Candidate.Source, precedence)) + .Where(vc => vc.Candidate.Modified - best.Candidate.Modified >= _freshnessThreshold) + .OrderByDescending(vc => vc.Candidate.Modified) + .ThenBy(vc => GetRank(vc.Candidate.Source, precedence)) + .FirstOrDefault(); + + if (freshnessOverride is not null) + { + best = freshnessOverride; + decisionReason = "freshness_override"; + } + } + + var sameRankCandidates = valueCandidates + .Where(vc => GetRank(vc.Candidate.Source, precedence) == GetRank(best.Candidate.Source, precedence)) + .ToList(); + + if (sameRankCandidates.Count > 1) + { + var tied = sameRankCandidates + .OrderBy(vc => vc.Value.Length) + .ThenBy(vc => vc.Value, StringComparer.Ordinal) + .ThenBy(vc => ComputeStableHash(vc.Value)) + .First(); + + if (!ReferenceEquals(tied, best)) + { + best = tied; + decisionReason = "tie_breaker"; + } + } + + var decision = new FieldDecision( + field, + best.Candidate.Source, + decisionReason, + best.Candidate.Modified, + consideredSources); + + return new FieldSelection(field, best.Value, best.Candidate, decisionReason, decision); + } + + private static void AddMergeProvenance( + HashSet provenanceSet, + FieldSelection selection, + DateTimeOffset recordedAt, + string fieldMask) + { + if (!selection.HasValue || selection.Winner is null) + { + return; + } + + var provenance = new AdvisoryProvenance( + selection.Winner.Source, + kind: "merge", + value: selection.Field, + recordedAt: recordedAt, + fieldMask: new[] { fieldMask }, + decisionReason: selection.DecisionReason); + + provenanceSet.Add(provenance); + } + + private static List BuildCandidates(Advisory? ghsa, Advisory? nvd, Advisory? osv) + { + var list = new List(capacity: 3); + if (ghsa is not null) + { + list.Add(CreateSnapshot(GhsaSource, ghsa)); + } + + if (nvd is not null) + { + list.Add(CreateSnapshot(NvdSource, nvd)); + } + + if (osv is not null) + { + list.Add(CreateSnapshot(OsvSource, osv)); + } + + return list; + } + + private static AdvisorySnapshot CreateSnapshot(string source, Advisory advisory) + { + var modified = advisory.Modified + ?? advisory.Published + ?? DateTimeOffset.UnixEpoch; + return new AdvisorySnapshot(source, advisory, modified); + } + + private static ImmutableDictionary GetPrecedence(string field) + { + if (FieldPrecedence.TryGetValue(field, out var order)) + { + return order + .Select((source, index) => (source, index)) + .ToImmutableDictionary(item => item.source, item => item.index, StringComparer.OrdinalIgnoreCase); + } + + return SourceOrder; + } + + private static int GetRank(string source, ImmutableDictionary precedence) + => precedence.TryGetValue(source, out var rank) ? rank : int.MaxValue; + + private static string ComputeStableHash(string value) + { + var bytes = Encoding.UTF8.GetBytes(value); + var hash = SHA256.HashData(bytes); + return Convert.ToHexString(hash); + } + + private sealed class FieldSelection + { + public FieldSelection(string field, T? value, AdvisorySnapshot? winner, string decisionReason, FieldDecision decision) + { + Field = field; + Value = value; + Winner = winner; + DecisionReason = decisionReason; + Decision = decision; + } + + public string Field { get; } + + public T? Value { get; } + + public AdvisorySnapshot? Winner { get; } + + public string DecisionReason { get; } + + public FieldDecision Decision { get; } + + public bool HasValue => Winner is not null; + + public static FieldSelection Empty { get; } = new FieldSelection( + string.Empty, + default, + null, + string.Empty, + new FieldDecision(string.Empty, null, string.Empty, null, ImmutableArray.Empty)); + } + + private sealed record AdvisorySnapshot(string Source, Advisory Advisory, DateTimeOffset Modified); + + private sealed record ValueCandidate(AdvisorySnapshot Candidate, string Value); + + private readonly record struct PackageSelection(AffectedPackage Package, string Source, DateTimeOffset Modified); + + private readonly record struct ReferenceSelection(AdvisoryReference Reference, string Source, DateTimeOffset Modified); + + private readonly record struct CreditSelection(AdvisoryCredit Credit, string Source, DateTimeOffset Modified); + + private readonly record struct MetricSelection(CvssMetric Metric, string Source, DateTimeOffset Modified); + + private readonly record struct CreditsMergeResult(ImmutableArray Credits, FieldDecision? Decision); + + private readonly record struct ReferencesMergeResult(ImmutableArray References, FieldDecision? Decision); + + private readonly record struct PackagesMergeResult( + ImmutableArray Packages, + IReadOnlyList Decisions, + IReadOnlyList AdditionalProvenance); + + private readonly record struct CvssMergeResult( + ImmutableArray Metrics, + string? CanonicalSeverity, + FieldDecision? Decision); +} diff --git a/src/StellaOps.Feedser.Core/Jobs/IJob.cs b/src/StellaOps.Feedser.Core/Jobs/IJob.cs new file mode 100644 index 00000000..8800c4ab --- /dev/null +++ b/src/StellaOps.Feedser.Core/Jobs/IJob.cs @@ -0,0 +1,6 @@ +namespace StellaOps.Feedser.Core.Jobs; + +public interface IJob +{ + Task ExecuteAsync(JobExecutionContext context, CancellationToken cancellationToken); +} diff --git a/src/StellaOps.Feedser.Core/Jobs/IJobCoordinator.cs b/src/StellaOps.Feedser.Core/Jobs/IJobCoordinator.cs new file mode 100644 index 00000000..ab3588d8 --- /dev/null +++ b/src/StellaOps.Feedser.Core/Jobs/IJobCoordinator.cs @@ -0,0 +1,18 @@ +namespace StellaOps.Feedser.Core.Jobs; + +public interface IJobCoordinator +{ + Task TriggerAsync(string kind, IReadOnlyDictionary? parameters, string trigger, CancellationToken cancellationToken); + + Task> GetDefinitionsAsync(CancellationToken cancellationToken); + + Task> GetRecentRunsAsync(string? kind, int limit, CancellationToken cancellationToken); + + Task> GetActiveRunsAsync(CancellationToken cancellationToken); + + Task GetRunAsync(Guid runId, CancellationToken cancellationToken); + + Task GetLastRunAsync(string kind, CancellationToken cancellationToken); + + Task> GetLastRunsAsync(IEnumerable kinds, CancellationToken cancellationToken); +} diff --git a/src/StellaOps.Feedser.Core/Jobs/IJobStore.cs b/src/StellaOps.Feedser.Core/Jobs/IJobStore.cs new file mode 100644 index 00000000..ee3914d7 --- /dev/null +++ b/src/StellaOps.Feedser.Core/Jobs/IJobStore.cs @@ -0,0 +1,20 @@ +namespace StellaOps.Feedser.Core.Jobs; + +public interface IJobStore +{ + Task CreateAsync(JobRunCreateRequest request, CancellationToken cancellationToken); + + Task TryStartAsync(Guid runId, DateTimeOffset startedAt, CancellationToken cancellationToken); + + Task TryCompleteAsync(Guid runId, JobRunCompletion completion, CancellationToken cancellationToken); + + Task FindAsync(Guid runId, CancellationToken cancellationToken); + + Task> GetRecentRunsAsync(string? kind, int limit, CancellationToken cancellationToken); + + Task> GetActiveRunsAsync(CancellationToken cancellationToken); + + Task GetLastRunAsync(string kind, CancellationToken cancellationToken); + + Task> GetLastRunsAsync(IEnumerable kinds, CancellationToken cancellationToken); +} diff --git a/src/StellaOps.Feedser.Core/Jobs/ILeaseStore.cs b/src/StellaOps.Feedser.Core/Jobs/ILeaseStore.cs new file mode 100644 index 00000000..802d4261 --- /dev/null +++ b/src/StellaOps.Feedser.Core/Jobs/ILeaseStore.cs @@ -0,0 +1,10 @@ +namespace StellaOps.Feedser.Core.Jobs; + +public interface ILeaseStore +{ + Task TryAcquireAsync(string key, string holder, TimeSpan leaseDuration, DateTimeOffset now, CancellationToken cancellationToken); + + Task HeartbeatAsync(string key, string holder, TimeSpan leaseDuration, DateTimeOffset now, CancellationToken cancellationToken); + + Task ReleaseAsync(string key, string holder, CancellationToken cancellationToken); +} diff --git a/src/StellaOps.Feedser.Core/Jobs/JobCoordinator.cs b/src/StellaOps.Feedser.Core/Jobs/JobCoordinator.cs new file mode 100644 index 00000000..a7e47b42 --- /dev/null +++ b/src/StellaOps.Feedser.Core/Jobs/JobCoordinator.cs @@ -0,0 +1,635 @@ +using System.Collections; +using System.Diagnostics; +using System.Security.Cryptography; +using System.Text; +using System.Text.Json; +using System.Globalization; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Options; + +namespace StellaOps.Feedser.Core.Jobs; + +public sealed class JobCoordinator : IJobCoordinator +{ + private readonly JobSchedulerOptions _options; + private readonly IJobStore _jobStore; + private readonly ILeaseStore _leaseStore; + private readonly IServiceScopeFactory _scopeFactory; + private readonly ILogger _logger; + private readonly ILoggerFactory _loggerFactory; + private readonly TimeProvider _timeProvider; + private readonly JobDiagnostics _diagnostics; + private readonly string _holderId; + + public JobCoordinator( + IOptions optionsAccessor, + IJobStore jobStore, + ILeaseStore leaseStore, + IServiceScopeFactory scopeFactory, + ILogger logger, + ILoggerFactory loggerFactory, + TimeProvider timeProvider, + JobDiagnostics diagnostics) + { + _options = (optionsAccessor ?? throw new ArgumentNullException(nameof(optionsAccessor))).Value; + _jobStore = jobStore ?? throw new ArgumentNullException(nameof(jobStore)); + _leaseStore = leaseStore ?? throw new ArgumentNullException(nameof(leaseStore)); + _scopeFactory = scopeFactory ?? throw new ArgumentNullException(nameof(scopeFactory)); + _logger = logger ?? throw new ArgumentNullException(nameof(logger)); + _loggerFactory = loggerFactory ?? throw new ArgumentNullException(nameof(loggerFactory)); + _timeProvider = timeProvider ?? throw new ArgumentNullException(nameof(timeProvider)); + _diagnostics = diagnostics ?? throw new ArgumentNullException(nameof(diagnostics)); + _holderId = BuildHolderId(); + } + + public async Task TriggerAsync(string kind, IReadOnlyDictionary? parameters, string trigger, CancellationToken cancellationToken) + { + using var triggerActivity = _diagnostics.StartTriggerActivity(kind, trigger); + + if (!_options.Definitions.TryGetValue(kind, out var definition)) + { + var result = JobTriggerResult.NotFound($"Job kind '{kind}' is not registered."); + triggerActivity?.SetStatus(ActivityStatusCode.Error, result.ErrorMessage); + triggerActivity?.SetTag("job.trigger.outcome", result.Outcome.ToString()); + _diagnostics.RecordTriggerRejected(kind, trigger, "not_found"); + return result; + } + + triggerActivity?.SetTag("job.enabled", definition.Enabled); + triggerActivity?.SetTag("job.timeout_seconds", definition.Timeout.TotalSeconds); + triggerActivity?.SetTag("job.lease_seconds", definition.LeaseDuration.TotalSeconds); + + if (!definition.Enabled) + { + var result = JobTriggerResult.Disabled($"Job kind '{kind}' is disabled."); + triggerActivity?.SetStatus(ActivityStatusCode.Ok, "disabled"); + triggerActivity?.SetTag("job.trigger.outcome", result.Outcome.ToString()); + _diagnostics.RecordTriggerRejected(kind, trigger, "disabled"); + return result; + } + + parameters ??= new Dictionary(); + + var parameterSnapshot = parameters.Count == 0 + ? new Dictionary(StringComparer.Ordinal) + : new Dictionary(parameters, StringComparer.Ordinal); + + if (!TryNormalizeParameters(parameterSnapshot, out var normalizedParameters, out var parameterError)) + { + var message = string.IsNullOrWhiteSpace(parameterError) + ? "Job trigger parameters contain unsupported values." + : parameterError; + triggerActivity?.SetStatus(ActivityStatusCode.Error, message); + triggerActivity?.SetTag("job.trigger.outcome", JobTriggerOutcome.InvalidParameters.ToString()); + _diagnostics.RecordTriggerRejected(kind, trigger, "invalid_parameters"); + return JobTriggerResult.InvalidParameters(message); + } + + parameterSnapshot = normalizedParameters; + + string? parametersHash; + try + { + parametersHash = JobParametersHasher.Compute(parameterSnapshot); + } + catch (Exception ex) + { + var message = $"Job trigger parameters cannot be serialized: {ex.Message}"; + triggerActivity?.SetStatus(ActivityStatusCode.Error, message); + triggerActivity?.SetTag("job.trigger.outcome", JobTriggerOutcome.InvalidParameters.ToString()); + _diagnostics.RecordTriggerRejected(kind, trigger, "invalid_parameters"); + _logger.LogWarning(ex, "Failed to serialize parameters for job {Kind}", kind); + return JobTriggerResult.InvalidParameters(message); + } + + triggerActivity?.SetTag("job.parameters_count", parameterSnapshot.Count); + + var now = _timeProvider.GetUtcNow(); + var leaseDuration = definition.LeaseDuration <= TimeSpan.Zero ? _options.DefaultLeaseDuration : definition.LeaseDuration; + + JobLease? lease = null; + try + { + lease = await _leaseStore.TryAcquireAsync(definition.LeaseKey, _holderId, leaseDuration, now, cancellationToken).ConfigureAwait(false); + if (lease is null) + { + var result = JobTriggerResult.AlreadyRunning($"Job '{kind}' is already running."); + triggerActivity?.SetStatus(ActivityStatusCode.Ok, "already_running"); + triggerActivity?.SetTag("job.trigger.outcome", result.Outcome.ToString()); + _diagnostics.RecordTriggerRejected(kind, trigger, "already_running"); + return result; + } + + var createdAt = _timeProvider.GetUtcNow(); + var request = new JobRunCreateRequest( + definition.Kind, + trigger, + parameterSnapshot, + parametersHash, + definition.Timeout, + leaseDuration, + createdAt); + + triggerActivity?.SetTag("job.parameters_hash", request.ParametersHash); + + var run = await _jobStore.CreateAsync(request, cancellationToken).ConfigureAwait(false); + var startedAt = _timeProvider.GetUtcNow(); + var started = await _jobStore.TryStartAsync(run.RunId, startedAt, cancellationToken).ConfigureAwait(false) ?? run; + + triggerActivity?.SetTag("job.run_id", started.RunId); + triggerActivity?.SetTag("job.created_at", createdAt.UtcDateTime); + triggerActivity?.SetTag("job.started_at", started.StartedAt?.UtcDateTime ?? startedAt.UtcDateTime); + + var linkedTokenSource = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken); + if (definition.Timeout > TimeSpan.Zero) + { + linkedTokenSource.CancelAfter(definition.Timeout); + } + + var capturedLease = lease ?? throw new InvalidOperationException("Lease acquisition returned null."); + try + { + _ = Task.Run(() => ExecuteJobAsync(definition, capturedLease, started, parameterSnapshot, trigger, linkedTokenSource), CancellationToken.None) + .ContinueWith(t => + { + if (t.Exception is not null) + { + _logger.LogError(t.Exception, "Unhandled job execution failure for {Kind}", definition.Kind); + } + }, + TaskContinuationOptions.OnlyOnFaulted | TaskContinuationOptions.ExecuteSynchronously); + lease = null; // released by background job execution + } + catch (Exception ex) + { + lease = capturedLease; // ensure outer finally releases if scheduling fails + triggerActivity?.SetStatus(ActivityStatusCode.Error, ex.Message); + triggerActivity?.SetTag("job.trigger.outcome", "exception"); + _diagnostics.RecordTriggerRejected(kind, trigger, "queue_failure"); + throw; + } + + var accepted = JobTriggerResult.Accepted(started); + _diagnostics.RecordTriggerAccepted(kind, trigger); + triggerActivity?.SetStatus(ActivityStatusCode.Ok); + triggerActivity?.SetTag("job.trigger.outcome", accepted.Outcome.ToString()); + return accepted; + } + catch (Exception ex) + { + triggerActivity?.SetStatus(ActivityStatusCode.Error, ex.Message); + triggerActivity?.SetTag("job.trigger.outcome", "exception"); + _diagnostics.RecordTriggerRejected(kind, trigger, "exception"); + throw; + } + finally + { + // Release handled by background execution path. If we failed before scheduling, release here. + if (lease is not null) + { + var releaseError = await TryReleaseLeaseAsync(lease, definition.Kind).ConfigureAwait(false); + if (releaseError is not null) + { + _logger.LogError(releaseError, "Failed to release lease {LeaseKey} for job {Kind}", lease.Key, definition.Kind); + } + } + } + } + + public Task> GetDefinitionsAsync(CancellationToken cancellationToken) + { + IReadOnlyList results = _options.Definitions.Values.OrderBy(x => x.Kind, StringComparer.Ordinal).ToArray(); + return Task.FromResult(results); + } + + public Task> GetRecentRunsAsync(string? kind, int limit, CancellationToken cancellationToken) + => _jobStore.GetRecentRunsAsync(kind, limit, cancellationToken); + + public Task> GetActiveRunsAsync(CancellationToken cancellationToken) + => _jobStore.GetActiveRunsAsync(cancellationToken); + + public Task GetRunAsync(Guid runId, CancellationToken cancellationToken) + => _jobStore.FindAsync(runId, cancellationToken); + + public Task GetLastRunAsync(string kind, CancellationToken cancellationToken) + => _jobStore.GetLastRunAsync(kind, cancellationToken); + + public Task> GetLastRunsAsync(IEnumerable kinds, CancellationToken cancellationToken) + => _jobStore.GetLastRunsAsync(kinds, cancellationToken); + + private static bool TryNormalizeParameters( + IReadOnlyDictionary source, + out Dictionary normalized, + out string? error) + { + if (source.Count == 0) + { + normalized = new Dictionary(StringComparer.Ordinal); + error = null; + return true; + } + + normalized = new Dictionary(source.Count, StringComparer.Ordinal); + foreach (var kvp in source) + { + if (string.IsNullOrWhiteSpace(kvp.Key)) + { + error = "Parameter keys must be non-empty strings."; + normalized = default!; + return false; + } + + try + { + normalized[kvp.Key] = NormalizeParameterValue(kvp.Value); + } + catch (Exception ex) + { + error = $"Parameter '{kvp.Key}' cannot be serialized: {ex.Message}"; + normalized = default!; + return false; + } + } + + error = null; + return true; + } + + private static object? NormalizeParameterValue(object? value) + { + if (value is null) + { + return null; + } + + switch (value) + { + case string or bool or double or decimal: + return value; + case byte or sbyte or short or ushort or int or long: + return Convert.ToInt64(value, CultureInfo.InvariantCulture); + case uint ui: + return Convert.ToInt64(ui); + case ulong ul when ul <= long.MaxValue: + return (long)ul; + case ulong ul: + return ul.ToString(CultureInfo.InvariantCulture); + case float f: + return (double)f; + case DateTime dt: + return dt.Kind == DateTimeKind.Utc ? dt : dt.ToUniversalTime(); + case DateTimeOffset dto: + return dto.ToUniversalTime(); + case TimeSpan ts: + return ts.ToString("c", CultureInfo.InvariantCulture); + case Guid guid: + return guid.ToString("D"); + case Enum enumValue: + return enumValue.ToString(); + case byte[] bytes: + return Convert.ToBase64String(bytes); + case JsonDocument document: + return NormalizeJsonElement(document.RootElement); + case JsonElement element: + return NormalizeJsonElement(element); + case IDictionary dictionary: + { + var nested = new SortedDictionary(StringComparer.Ordinal); + foreach (DictionaryEntry entry in dictionary) + { + if (entry.Key is not string key || string.IsNullOrWhiteSpace(key)) + { + throw new InvalidOperationException("Nested dictionary keys must be non-empty strings."); + } + + nested[key] = NormalizeParameterValue(entry.Value); + } + + return nested; + } + case IEnumerable enumerable when value is not string: + { + var list = new List(); + foreach (var item in enumerable) + { + list.Add(NormalizeParameterValue(item)); + } + + return list; + } + default: + throw new InvalidOperationException($"Unsupported parameter value of type '{value.GetType().FullName}'."); + } + } + + private static object? NormalizeJsonElement(JsonElement element) + { + return element.ValueKind switch + { + JsonValueKind.Null => null, + JsonValueKind.String => element.GetString(), + JsonValueKind.True => true, + JsonValueKind.False => false, + JsonValueKind.Number => element.TryGetInt64(out var l) + ? l + : element.TryGetDecimal(out var dec) + ? dec + : element.GetDouble(), + JsonValueKind.Object => NormalizeJsonObject(element), + JsonValueKind.Array => NormalizeJsonArray(element), + _ => throw new InvalidOperationException($"Unsupported JSON value '{element.ValueKind}'."), + }; + } + + private static SortedDictionary NormalizeJsonObject(JsonElement element) + { + var result = new SortedDictionary(StringComparer.Ordinal); + foreach (var property in element.EnumerateObject()) + { + result[property.Name] = NormalizeJsonElement(property.Value); + } + + return result; + } + + private static List NormalizeJsonArray(JsonElement element) + { + var items = new List(); + foreach (var item in element.EnumerateArray()) + { + items.Add(NormalizeJsonElement(item)); + } + + return items; + } + + private async Task CompleteRunAsync(Guid runId, JobRunStatus status, string? error, CancellationToken cancellationToken) + { + var completedAt = _timeProvider.GetUtcNow(); + var completion = new JobRunCompletion(status, completedAt, error); + return await _jobStore.TryCompleteAsync(runId, completion, cancellationToken).ConfigureAwait(false); + } + + private TimeSpan? ResolveDuration(JobRunSnapshot original, JobRunSnapshot? completed) + { + if (completed?.Duration is { } duration) + { + return duration; + } + + var startedAt = completed?.StartedAt ?? original.StartedAt ?? original.CreatedAt; + var completedAt = completed?.CompletedAt ?? _timeProvider.GetUtcNow(); + var elapsed = completedAt - startedAt; + return elapsed >= TimeSpan.Zero ? elapsed : null; + } + + private static async Task ObserveLeaseTaskAsync(Task heartbeatTask) + { + try + { + await heartbeatTask.ConfigureAwait(false); + return null; + } + catch (OperationCanceledException) + { + return null; + } + catch (Exception ex) + { + return ex; + } + } + + private async Task TryReleaseLeaseAsync(JobLease lease, string kind) + { + try + { + await _leaseStore.ReleaseAsync(lease.Key, _holderId, CancellationToken.None).ConfigureAwait(false); + return null; + } + catch (Exception ex) + { + return new LeaseMaintenanceException($"Failed to release lease for job '{kind}'.", ex); + } + } + + private static Exception? CombineLeaseExceptions(Exception? first, Exception? second) + { + if (first is null) + { + return second; + } + + if (second is null) + { + return first; + } + + return new AggregateException(first, second); + } + + private async Task ExecuteJobAsync( + JobDefinition definition, + JobLease lease, + JobRunSnapshot run, + IReadOnlyDictionary parameters, + string trigger, + CancellationTokenSource linkedTokenSource) + { + using (linkedTokenSource) + { + var cancellationToken = linkedTokenSource.Token; + using var heartbeatCts = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken); + var heartbeatTask = MaintainLeaseAsync(definition, lease, heartbeatCts.Token); + + using var activity = _diagnostics.StartExecutionActivity(run.Kind, trigger, run.RunId); + activity?.SetTag("job.timeout_seconds", definition.Timeout.TotalSeconds); + activity?.SetTag("job.lease_seconds", definition.LeaseDuration.TotalSeconds); + activity?.SetTag("job.parameters_count", parameters.Count); + activity?.SetTag("job.created_at", run.CreatedAt.UtcDateTime); + activity?.SetTag("job.started_at", (run.StartedAt ?? run.CreatedAt).UtcDateTime); + activity?.SetTag("job.parameters_hash", run.ParametersHash); + + _diagnostics.RecordRunStarted(run.Kind); + + JobRunStatus finalStatus = JobRunStatus.Succeeded; + string? error = null; + Exception? executionException = null; + JobRunSnapshot? completedSnapshot = null; + Exception? leaseException = null; + + try + { + using var scope = _scopeFactory.CreateScope(); + var job = (IJob)scope.ServiceProvider.GetRequiredService(definition.JobType); + var jobLogger = _loggerFactory.CreateLogger(definition.JobType); + + var context = new JobExecutionContext( + run.RunId, + run.Kind, + trigger, + parameters, + scope.ServiceProvider, + _timeProvider, + jobLogger); + + await job.ExecuteAsync(context, cancellationToken).ConfigureAwait(false); + } + catch (OperationCanceledException oce) + { + finalStatus = JobRunStatus.Cancelled; + error = oce.Message; + executionException = oce; + } + catch (Exception ex) + { + finalStatus = JobRunStatus.Failed; + error = ex.ToString(); + executionException = ex; + } + finally + { + heartbeatCts.Cancel(); + + leaseException = await ObserveLeaseTaskAsync(heartbeatTask).ConfigureAwait(false); + + var releaseException = await TryReleaseLeaseAsync(lease, definition.Kind).ConfigureAwait(false); + leaseException = CombineLeaseExceptions(leaseException, releaseException); + + if (leaseException is not null) + { + var leaseMessage = $"Lease maintenance failed: {leaseException.GetType().Name}: {leaseException.Message}"; + if (finalStatus != JobRunStatus.Failed) + { + finalStatus = JobRunStatus.Failed; + error = leaseMessage; + executionException = leaseException; + } + else + { + error = string.IsNullOrWhiteSpace(error) + ? leaseMessage + : $"{error}{Environment.NewLine}{leaseMessage}"; + executionException = executionException is null + ? leaseException + : new AggregateException(executionException, leaseException); + } + } + } + + completedSnapshot = await CompleteRunAsync(run.RunId, finalStatus, error, CancellationToken.None).ConfigureAwait(false); + + if (!string.IsNullOrWhiteSpace(error)) + { + activity?.SetTag("job.error", error); + } + + activity?.SetTag("job.status", finalStatus.ToString()); + + var completedDuration = ResolveDuration(run, completedSnapshot); + if (completedDuration.HasValue) + { + activity?.SetTag("job.duration_seconds", completedDuration.Value.TotalSeconds); + } + + switch (finalStatus) + { + case JobRunStatus.Succeeded: + activity?.SetStatus(ActivityStatusCode.Ok); + _logger.LogInformation("Job {Kind} run {RunId} succeeded", run.Kind, run.RunId); + break; + case JobRunStatus.Cancelled: + activity?.SetStatus(ActivityStatusCode.Ok, "cancelled"); + _logger.LogWarning(executionException, "Job {Kind} run {RunId} cancelled", run.Kind, run.RunId); + break; + case JobRunStatus.Failed: + activity?.SetStatus(ActivityStatusCode.Error, executionException?.Message ?? error); + _logger.LogError(executionException, "Job {Kind} run {RunId} failed", run.Kind, run.RunId); + break; + } + + _diagnostics.RecordRunCompleted(run.Kind, finalStatus, completedDuration, error); + } + } + + private async Task MaintainLeaseAsync(JobDefinition definition, JobLease lease, CancellationToken cancellationToken) + { + var leaseDuration = lease.LeaseDuration <= TimeSpan.Zero ? _options.DefaultLeaseDuration : lease.LeaseDuration; + var delay = TimeSpan.FromMilliseconds(Math.Max(1000, leaseDuration.TotalMilliseconds / 2)); + + while (!cancellationToken.IsCancellationRequested) + { + try + { + await Task.Delay(delay, cancellationToken).ConfigureAwait(false); + } + catch (TaskCanceledException) + { + break; + } + + var now = _timeProvider.GetUtcNow(); + try + { + await _leaseStore.HeartbeatAsync(definition.LeaseKey, _holderId, leaseDuration, now, cancellationToken).ConfigureAwait(false); + } + catch (OperationCanceledException) + { + break; + } + catch (Exception ex) + { + throw new LeaseMaintenanceException($"Failed to heartbeat lease for job '{definition.Kind}'.", ex); + } + } + } + + private static string BuildHolderId() + { + var machine = Environment.MachineName; + var processId = Environment.ProcessId; + return $"{machine}:{processId}"; + } +} + +internal sealed class LeaseMaintenanceException : Exception +{ + public LeaseMaintenanceException(string message, Exception innerException) + : base(message, innerException) + { + } +} + +internal static class JobParametersHasher +{ + internal static readonly JsonSerializerOptions SerializerOptions = new() + { + PropertyNamingPolicy = JsonNamingPolicy.CamelCase, + WriteIndented = false, + }; + + public static string? Compute(IReadOnlyDictionary parameters) + { + if (parameters is null || parameters.Count == 0) + { + return null; + } + + var canonicalJson = JsonSerializer.Serialize(Sort(parameters), SerializerOptions); + var bytes = Encoding.UTF8.GetBytes(canonicalJson); + var hash = SHA256.HashData(bytes); + return Convert.ToHexString(hash).ToLowerInvariant(); + } + + private static SortedDictionary Sort(IReadOnlyDictionary parameters) + { + var sorted = new SortedDictionary(StringComparer.Ordinal); + foreach (var kvp in parameters) + { + sorted[kvp.Key] = kvp.Value; + } + + return sorted; + } +} diff --git a/src/StellaOps.Feedser.Core/Jobs/JobDefinition.cs b/src/StellaOps.Feedser.Core/Jobs/JobDefinition.cs new file mode 100644 index 00000000..1e56f674 --- /dev/null +++ b/src/StellaOps.Feedser.Core/Jobs/JobDefinition.cs @@ -0,0 +1,12 @@ +namespace StellaOps.Feedser.Core.Jobs; + +public sealed record JobDefinition( + string Kind, + Type JobType, + TimeSpan Timeout, + TimeSpan LeaseDuration, + string? CronExpression, + bool Enabled) +{ + public string LeaseKey => $"job:{Kind}"; +} diff --git a/src/StellaOps.Feedser.Core/Jobs/JobDiagnostics.cs b/src/StellaOps.Feedser.Core/Jobs/JobDiagnostics.cs new file mode 100644 index 00000000..d90ef781 --- /dev/null +++ b/src/StellaOps.Feedser.Core/Jobs/JobDiagnostics.cs @@ -0,0 +1,171 @@ +using System.Diagnostics; +using System.Diagnostics.Metrics; + +namespace StellaOps.Feedser.Core.Jobs; + +public sealed class JobDiagnostics : IDisposable +{ + public const string ActivitySourceName = "StellaOps.Feedser.Jobs"; + public const string MeterName = "StellaOps.Feedser.Jobs"; + public const string TriggerActivityName = "feedser.job.trigger"; + public const string ExecuteActivityName = "feedser.job.execute"; + public const string SchedulerActivityName = "feedser.scheduler.evaluate"; + + private readonly Counter _triggersAccepted; + private readonly Counter _triggersRejected; + private readonly Counter _runsCompleted; + private readonly UpDownCounter _runsActive; + private readonly Histogram _runDurationSeconds; + private readonly Histogram _schedulerSkewMilliseconds; + + public JobDiagnostics() + { + ActivitySource = new ActivitySource(ActivitySourceName); + Meter = new Meter(MeterName); + + _triggersAccepted = Meter.CreateCounter( + name: "feedser.jobs.triggers.accepted", + unit: "count", + description: "Number of job trigger requests accepted for execution."); + + _triggersRejected = Meter.CreateCounter( + name: "feedser.jobs.triggers.rejected", + unit: "count", + description: "Number of job trigger requests rejected or ignored by the coordinator."); + + _runsCompleted = Meter.CreateCounter( + name: "feedser.jobs.runs.completed", + unit: "count", + description: "Number of job executions that have finished grouped by outcome."); + + _runsActive = Meter.CreateUpDownCounter( + name: "feedser.jobs.runs.active", + unit: "count", + description: "Current number of running job executions."); + + _runDurationSeconds = Meter.CreateHistogram( + name: "feedser.jobs.runs.duration", + unit: "s", + description: "Distribution of job execution durations in seconds."); + + _schedulerSkewMilliseconds = Meter.CreateHistogram( + name: "feedser.scheduler.skew", + unit: "ms", + description: "Difference between the intended and actual scheduler fire time in milliseconds."); + } + + public ActivitySource ActivitySource { get; } + + public Meter Meter { get; } + + public Activity? StartTriggerActivity(string kind, string trigger) + { + var activity = ActivitySource.StartActivity(TriggerActivityName, ActivityKind.Internal); + if (activity is not null) + { + activity.SetTag("job.kind", kind); + activity.SetTag("job.trigger", trigger); + } + + return activity; + } + + public Activity? StartSchedulerActivity(string kind, DateTimeOffset scheduledFor, DateTimeOffset invokedAt) + { + var activity = ActivitySource.StartActivity(SchedulerActivityName, ActivityKind.Internal); + if (activity is not null) + { + activity.SetTag("job.kind", kind); + activity.SetTag("job.scheduled_for", scheduledFor.UtcDateTime); + activity.SetTag("job.invoked_at", invokedAt.UtcDateTime); + activity.SetTag("job.scheduler_delay_ms", (invokedAt - scheduledFor).TotalMilliseconds); + } + + return activity; + } + + public Activity? StartExecutionActivity(string kind, string trigger, Guid runId) + { + var activity = ActivitySource.StartActivity(ExecuteActivityName, ActivityKind.Internal); + if (activity is not null) + { + activity.SetTag("job.kind", kind); + activity.SetTag("job.trigger", trigger); + activity.SetTag("job.run_id", runId); + } + + return activity; + } + + public void RecordTriggerAccepted(string kind, string trigger) + { + var tags = new TagList + { + { "job.kind", kind }, + { "job.trigger", trigger }, + }; + _triggersAccepted.Add(1, tags); + } + + public void RecordTriggerRejected(string kind, string trigger, string reason) + { + var tags = new TagList + { + { "job.kind", kind }, + { "job.trigger", trigger }, + { "job.reason", reason }, + }; + _triggersRejected.Add(1, tags); + } + + public void RecordRunStarted(string kind) + { + var tags = new TagList { { "job.kind", kind } }; + _runsActive.Add(1, tags); + } + + public void RecordRunCompleted(string kind, JobRunStatus status, TimeSpan? duration, string? error) + { + var outcome = status.ToString(); + + var completionTags = new TagList + { + { "job.kind", kind }, + { "job.status", outcome }, + }; + + if (!string.IsNullOrWhiteSpace(error)) + { + completionTags.Add("job.error", error); + } + + _runsCompleted.Add(1, completionTags); + + var activeTags = new TagList { { "job.kind", kind } }; + _runsActive.Add(-1, activeTags); + + if (duration.HasValue) + { + var seconds = Math.Max(duration.Value.TotalSeconds, 0d); + var durationTags = new TagList + { + { "job.kind", kind }, + { "job.status", outcome }, + }; + _runDurationSeconds.Record(seconds, durationTags); + } + } + + public void RecordSchedulerSkew(string kind, DateTimeOffset scheduledFor, DateTimeOffset invokedAt) + { + var skew = (invokedAt - scheduledFor).TotalMilliseconds; + var tags = new TagList { { "job.kind", kind } }; + _schedulerSkewMilliseconds.Record(skew, tags); + } + + public void Dispose() + { + ActivitySource.Dispose(); + Meter.Dispose(); + } +} diff --git a/src/StellaOps.Feedser.Core/Jobs/JobExecutionContext.cs b/src/StellaOps.Feedser.Core/Jobs/JobExecutionContext.cs new file mode 100644 index 00000000..24141695 --- /dev/null +++ b/src/StellaOps.Feedser.Core/Jobs/JobExecutionContext.cs @@ -0,0 +1,42 @@ +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Logging; + +namespace StellaOps.Feedser.Core.Jobs; + +public sealed class JobExecutionContext +{ + public JobExecutionContext( + Guid runId, + string kind, + string trigger, + IReadOnlyDictionary parameters, + IServiceProvider services, + TimeProvider timeProvider, + ILogger logger) + { + RunId = runId; + Kind = kind; + Trigger = trigger; + Parameters = parameters ?? throw new ArgumentNullException(nameof(parameters)); + Services = services ?? throw new ArgumentNullException(nameof(services)); + TimeProvider = timeProvider ?? throw new ArgumentNullException(nameof(timeProvider)); + Logger = logger ?? throw new ArgumentNullException(nameof(logger)); + } + + public Guid RunId { get; } + + public string Kind { get; } + + public string Trigger { get; } + + public IReadOnlyDictionary Parameters { get; } + + public IServiceProvider Services { get; } + + public TimeProvider TimeProvider { get; } + + public ILogger Logger { get; } + + public T GetRequiredService() where T : notnull + => Services.GetRequiredService(); +} diff --git a/src/StellaOps.Feedser.Core/Jobs/JobLease.cs b/src/StellaOps.Feedser.Core/Jobs/JobLease.cs new file mode 100644 index 00000000..2f01509e --- /dev/null +++ b/src/StellaOps.Feedser.Core/Jobs/JobLease.cs @@ -0,0 +1,9 @@ +namespace StellaOps.Feedser.Core.Jobs; + +public sealed record JobLease( + string Key, + string Holder, + DateTimeOffset AcquiredAt, + DateTimeOffset HeartbeatAt, + TimeSpan LeaseDuration, + DateTimeOffset TtlAt); diff --git a/src/StellaOps.Feedser.Core/Jobs/JobPluginRegistrationExtensions.cs b/src/StellaOps.Feedser.Core/Jobs/JobPluginRegistrationExtensions.cs new file mode 100644 index 00000000..b60f2608 --- /dev/null +++ b/src/StellaOps.Feedser.Core/Jobs/JobPluginRegistrationExtensions.cs @@ -0,0 +1,128 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Reflection; +using Microsoft.Extensions.Configuration; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Logging; +using StellaOps.DependencyInjection; +using StellaOps.Plugin.Hosting; + +namespace StellaOps.Feedser.Core.Jobs; + +public static class JobPluginRegistrationExtensions +{ + public static IServiceCollection RegisterJobPluginRoutines( + this IServiceCollection services, + IConfiguration configuration, + PluginHostOptions options, + ILogger? logger = null) + { + ArgumentNullException.ThrowIfNull(services); + ArgumentNullException.ThrowIfNull(configuration); + ArgumentNullException.ThrowIfNull(options); + + var loadResult = PluginHost.LoadPlugins(options, logger); + + if (!services.Any(sd => sd.ServiceType == typeof(PluginHostResult))) + { + services.AddSingleton(loadResult); + } + + var currentServices = services; + var seenRoutineTypes = new HashSet(StringComparer.Ordinal); + + foreach (var plugin in loadResult.Plugins) + { + foreach (var routineType in GetRoutineTypes(plugin.Assembly)) + { + if (!typeof(IDependencyInjectionRoutine).IsAssignableFrom(routineType)) + { + continue; + } + + if (routineType.IsInterface || routineType.IsAbstract) + { + continue; + } + + var routineKey = routineType.FullName ?? routineType.Name; + if (!seenRoutineTypes.Add(routineKey)) + { + continue; + } + + IDependencyInjectionRoutine? routineInstance; + try + { + routineInstance = Activator.CreateInstance(routineType) as IDependencyInjectionRoutine; + } + catch (Exception ex) + { + logger?.LogWarning( + ex, + "Failed to create dependency injection routine {Routine} from plugin {Plugin}.", + routineType.FullName ?? routineType.Name, + plugin.Assembly.FullName ?? plugin.AssemblyPath); + continue; + } + + if (routineInstance is null) + { + continue; + } + + try + { + var updated = routineInstance.Register(currentServices, configuration); + if (updated is not null && !ReferenceEquals(updated, currentServices)) + { + currentServices = updated; + } + } + catch (Exception ex) + { + logger?.LogError( + ex, + "Dependency injection routine {Routine} from plugin {Plugin} threw during registration.", + routineType.FullName ?? routineType.Name, + plugin.Assembly.FullName ?? plugin.AssemblyPath); + } + } + } + + if (loadResult.MissingOrderedPlugins.Count > 0) + { + logger?.LogWarning( + "Missing ordered plugin(s): {Missing}", + string.Join(", ", loadResult.MissingOrderedPlugins)); + } + + return currentServices; + } + + private static IEnumerable GetRoutineTypes(Assembly assembly) + { + if (assembly is null) + { + yield break; + } + + Type[] types; + try + { + types = assembly.GetTypes(); + } + catch (ReflectionTypeLoadException ex) + { + types = ex.Types.Where(static t => t is not null)! + .Select(static t => t!) + .ToArray(); + } + + foreach (var type in types) + { + yield return type; + } + } +} diff --git a/src/StellaOps.Feedser.Core/Jobs/JobRunCompletion.cs b/src/StellaOps.Feedser.Core/Jobs/JobRunCompletion.cs new file mode 100644 index 00000000..7018dba5 --- /dev/null +++ b/src/StellaOps.Feedser.Core/Jobs/JobRunCompletion.cs @@ -0,0 +1,6 @@ +namespace StellaOps.Feedser.Core.Jobs; + +public sealed record JobRunCompletion( + JobRunStatus Status, + DateTimeOffset CompletedAt, + string? Error); diff --git a/src/StellaOps.Feedser.Core/Jobs/JobRunCreateRequest.cs b/src/StellaOps.Feedser.Core/Jobs/JobRunCreateRequest.cs new file mode 100644 index 00000000..bf576559 --- /dev/null +++ b/src/StellaOps.Feedser.Core/Jobs/JobRunCreateRequest.cs @@ -0,0 +1,10 @@ +namespace StellaOps.Feedser.Core.Jobs; + +public sealed record JobRunCreateRequest( + string Kind, + string Trigger, + IReadOnlyDictionary Parameters, + string? ParametersHash, + TimeSpan? Timeout, + TimeSpan? LeaseDuration, + DateTimeOffset CreatedAt); diff --git a/src/StellaOps.Feedser.Core/Jobs/JobRunSnapshot.cs b/src/StellaOps.Feedser.Core/Jobs/JobRunSnapshot.cs new file mode 100644 index 00000000..7ef3099b --- /dev/null +++ b/src/StellaOps.Feedser.Core/Jobs/JobRunSnapshot.cs @@ -0,0 +1,21 @@ +namespace StellaOps.Feedser.Core.Jobs; + +/// +/// Immutable projection of a job run as stored in persistence. +/// +public sealed record JobRunSnapshot( + Guid RunId, + string Kind, + JobRunStatus Status, + DateTimeOffset CreatedAt, + DateTimeOffset? StartedAt, + DateTimeOffset? CompletedAt, + string Trigger, + string? ParametersHash, + string? Error, + TimeSpan? Timeout, + TimeSpan? LeaseDuration, + IReadOnlyDictionary Parameters) +{ + public TimeSpan? Duration => StartedAt is null || CompletedAt is null ? null : CompletedAt - StartedAt; +} diff --git a/src/StellaOps.Feedser.Core/Jobs/JobRunStatus.cs b/src/StellaOps.Feedser.Core/Jobs/JobRunStatus.cs new file mode 100644 index 00000000..a6871f22 --- /dev/null +++ b/src/StellaOps.Feedser.Core/Jobs/JobRunStatus.cs @@ -0,0 +1,10 @@ +namespace StellaOps.Feedser.Core.Jobs; + +public enum JobRunStatus +{ + Pending, + Running, + Succeeded, + Failed, + Cancelled, +} diff --git a/src/StellaOps.Feedser.Core/Jobs/JobSchedulerBuilder.cs b/src/StellaOps.Feedser.Core/Jobs/JobSchedulerBuilder.cs new file mode 100644 index 00000000..4c871ada --- /dev/null +++ b/src/StellaOps.Feedser.Core/Jobs/JobSchedulerBuilder.cs @@ -0,0 +1,47 @@ +using System; +using Microsoft.Extensions.DependencyInjection; + +namespace StellaOps.Feedser.Core.Jobs; + +public sealed class JobSchedulerBuilder +{ + private readonly IServiceCollection _services; + + public JobSchedulerBuilder(IServiceCollection services) + { + _services = services ?? throw new ArgumentNullException(nameof(services)); + } + + public JobSchedulerBuilder AddJob( + string kind, + string? cronExpression = null, + TimeSpan? timeout = null, + TimeSpan? leaseDuration = null, + bool enabled = true) + where TJob : class, IJob + { + ArgumentException.ThrowIfNullOrEmpty(kind); + + _services.AddTransient(); + _services.Configure(options => + { + if (options.Definitions.ContainsKey(kind)) + { + throw new InvalidOperationException($"Job '{kind}' is already registered."); + } + + var resolvedTimeout = timeout ?? options.DefaultTimeout; + var resolvedLease = leaseDuration ?? options.DefaultLeaseDuration; + + options.Definitions.Add(kind, new JobDefinition( + kind, + typeof(TJob), + resolvedTimeout, + resolvedLease, + cronExpression, + enabled)); + }); + + return this; + } +} diff --git a/src/StellaOps.Feedser.Core/Jobs/JobSchedulerHostedService.cs b/src/StellaOps.Feedser.Core/Jobs/JobSchedulerHostedService.cs new file mode 100644 index 00000000..7f74f378 --- /dev/null +++ b/src/StellaOps.Feedser.Core/Jobs/JobSchedulerHostedService.cs @@ -0,0 +1,165 @@ +using Cronos; +using System.Diagnostics; +using Microsoft.Extensions.Hosting; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Options; + +namespace StellaOps.Feedser.Core.Jobs; + +/// +/// Background service that evaluates cron expressions for registered jobs and triggers them. +/// +public sealed class JobSchedulerHostedService : BackgroundService +{ + private readonly IJobCoordinator _coordinator; + private readonly JobSchedulerOptions _options; + private readonly ILogger _logger; + private readonly TimeProvider _timeProvider; + private readonly JobDiagnostics _diagnostics; + private readonly Dictionary _cronExpressions = new(StringComparer.Ordinal); + private readonly Dictionary _nextOccurrences = new(StringComparer.Ordinal); + + public JobSchedulerHostedService( + IJobCoordinator coordinator, + IOptions optionsAccessor, + ILogger logger, + TimeProvider timeProvider, + JobDiagnostics diagnostics) + { + _coordinator = coordinator ?? throw new ArgumentNullException(nameof(coordinator)); + _options = (optionsAccessor ?? throw new ArgumentNullException(nameof(optionsAccessor))).Value; + _logger = logger ?? throw new ArgumentNullException(nameof(logger)); + _timeProvider = timeProvider ?? throw new ArgumentNullException(nameof(timeProvider)); + _diagnostics = diagnostics ?? throw new ArgumentNullException(nameof(diagnostics)); + + foreach (var definition in _options.Definitions.Values) + { + if (string.IsNullOrWhiteSpace(definition.CronExpression)) + { + continue; + } + + try + { + var cron = CronExpression.Parse(definition.CronExpression!, CronFormat.Standard); + _cronExpressions[definition.Kind] = cron; + } + catch (CronFormatException ex) + { + _logger.LogError(ex, "Invalid cron expression '{Cron}' for job {Kind}", definition.CronExpression, definition.Kind); + } + } + } + + protected override async Task ExecuteAsync(CancellationToken stoppingToken) + { + if (_cronExpressions.Count == 0) + { + _logger.LogInformation("No cron-based jobs registered; scheduler idle."); + await Task.Delay(Timeout.Infinite, stoppingToken).ConfigureAwait(false); + return; + } + + while (!stoppingToken.IsCancellationRequested) + { + var now = _timeProvider.GetUtcNow(); + var nextWake = now.AddMinutes(5); // default sleep when nothing scheduled + + foreach (var (kind, cron) in _cronExpressions) + { + if (!_options.Definitions.TryGetValue(kind, out var definition) || !definition.Enabled) + { + continue; + } + + var next = GetNextOccurrence(kind, cron, now); + if (next <= now.AddMilliseconds(500)) + { + _ = TriggerJobAsync(kind, next, stoppingToken); + _nextOccurrences[kind] = GetNextOccurrence(kind, cron, now.AddSeconds(1)); + next = _nextOccurrences[kind]; + } + + if (next < nextWake) + { + nextWake = next; + } + } + + var delay = nextWake - now; + if (delay < TimeSpan.FromSeconds(1)) + { + delay = TimeSpan.FromSeconds(1); + } + + try + { + await Task.Delay(delay, stoppingToken).ConfigureAwait(false); + } + catch (TaskCanceledException) + { + break; + } + } + } + + private DateTimeOffset GetNextOccurrence(string kind, CronExpression cron, DateTimeOffset reference) + { + if (_nextOccurrences.TryGetValue(kind, out var cached) && cached > reference) + { + return cached; + } + + var next = cron.GetNextOccurrence(reference.UtcDateTime, TimeZoneInfo.Utc); + if (next is null) + { + // No future occurrence; schedule far in future to avoid tight loop. + next = reference.UtcDateTime.AddYears(100); + } + + var nextUtc = DateTime.SpecifyKind(next.Value, DateTimeKind.Utc); + var offset = new DateTimeOffset(nextUtc); + _nextOccurrences[kind] = offset; + return offset; + } + + private async Task TriggerJobAsync(string kind, DateTimeOffset scheduledFor, CancellationToken stoppingToken) + { + var invokedAt = _timeProvider.GetUtcNow(); + _diagnostics.RecordSchedulerSkew(kind, scheduledFor, invokedAt); + + using var activity = _diagnostics.StartSchedulerActivity(kind, scheduledFor, invokedAt); + try + { + var result = await _coordinator.TriggerAsync(kind, parameters: null, trigger: "scheduler", stoppingToken).ConfigureAwait(false); + activity?.SetTag("job.trigger.outcome", result.Outcome.ToString()); + if (result.Run is not null) + { + activity?.SetTag("job.run_id", result.Run.RunId); + } + if (!string.IsNullOrWhiteSpace(result.ErrorMessage)) + { + activity?.SetTag("job.trigger.error", result.ErrorMessage); + } + + if (result.Outcome == JobTriggerOutcome.Accepted) + { + activity?.SetStatus(ActivityStatusCode.Ok); + } + else + { + activity?.SetStatus(ActivityStatusCode.Ok, result.Outcome.ToString()); + } + + if (result.Outcome != JobTriggerOutcome.Accepted) + { + _logger.LogDebug("Scheduler trigger for {Kind} resulted in {Outcome}", kind, result.Outcome); + } + } + catch (Exception ex) when (!stoppingToken.IsCancellationRequested) + { + activity?.SetStatus(ActivityStatusCode.Error, ex.Message); + _logger.LogError(ex, "Cron trigger for job {Kind} failed", kind); + } + } +} diff --git a/src/StellaOps.Feedser.Core/Jobs/JobSchedulerOptions.cs b/src/StellaOps.Feedser.Core/Jobs/JobSchedulerOptions.cs new file mode 100644 index 00000000..f3f22186 --- /dev/null +++ b/src/StellaOps.Feedser.Core/Jobs/JobSchedulerOptions.cs @@ -0,0 +1,12 @@ +namespace StellaOps.Feedser.Core.Jobs; + +public sealed class JobSchedulerOptions +{ + public static JobSchedulerOptions Empty { get; } = new(); + + public IDictionary Definitions { get; } = new Dictionary(StringComparer.Ordinal); + + public TimeSpan DefaultTimeout { get; set; } = TimeSpan.FromMinutes(15); + + public TimeSpan DefaultLeaseDuration { get; set; } = TimeSpan.FromMinutes(5); +} diff --git a/src/StellaOps.Feedser.Core/Jobs/JobTriggerResult.cs b/src/StellaOps.Feedser.Core/Jobs/JobTriggerResult.cs new file mode 100644 index 00000000..49e1d60b --- /dev/null +++ b/src/StellaOps.Feedser.Core/Jobs/JobTriggerResult.cs @@ -0,0 +1,40 @@ +namespace StellaOps.Feedser.Core.Jobs; + +public enum JobTriggerOutcome +{ + Accepted, + NotFound, + Disabled, + AlreadyRunning, + LeaseRejected, + InvalidParameters, + Failed, + Cancelled, +} + +public sealed record JobTriggerResult(JobTriggerOutcome Outcome, JobRunSnapshot? Run, string? ErrorMessage) +{ + public static JobTriggerResult Accepted(JobRunSnapshot run) + => new(JobTriggerOutcome.Accepted, run, null); + + public static JobTriggerResult NotFound(string message) + => new(JobTriggerOutcome.NotFound, null, message); + + public static JobTriggerResult Disabled(string message) + => new(JobTriggerOutcome.Disabled, null, message); + + public static JobTriggerResult AlreadyRunning(string message) + => new(JobTriggerOutcome.AlreadyRunning, null, message); + + public static JobTriggerResult LeaseRejected(string message) + => new(JobTriggerOutcome.LeaseRejected, null, message); + + public static JobTriggerResult InvalidParameters(string message) + => new(JobTriggerOutcome.InvalidParameters, null, message); + + public static JobTriggerResult Failed(JobRunSnapshot run, string error) + => new(JobTriggerOutcome.Failed, run, error); + + public static JobTriggerResult Cancelled(JobRunSnapshot run, string error) + => new(JobTriggerOutcome.Cancelled, run, error); +} diff --git a/src/StellaOps.Feedser.Core/Jobs/ServiceCollectionExtensions.cs b/src/StellaOps.Feedser.Core/Jobs/ServiceCollectionExtensions.cs new file mode 100644 index 00000000..99e98c96 --- /dev/null +++ b/src/StellaOps.Feedser.Core/Jobs/ServiceCollectionExtensions.cs @@ -0,0 +1,27 @@ +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.DependencyInjection.Extensions; +using Microsoft.Extensions.Options; + +namespace StellaOps.Feedser.Core.Jobs; + +public static class JobServiceCollectionExtensions +{ + public static JobSchedulerBuilder AddJobScheduler(this IServiceCollection services, Action? configure = null) + { + ArgumentNullException.ThrowIfNull(services); + + var optionsBuilder = services.AddOptions(); + if (configure is not null) + { + optionsBuilder.Configure(configure); + } + + services.AddSingleton(sp => sp.GetRequiredService>().Value); + services.AddSingleton(); + services.TryAddSingleton(TimeProvider.System); + services.AddSingleton(); + services.AddHostedService(); + + return new JobSchedulerBuilder(services); + } +} diff --git a/src/StellaOps.Feedser.Core/StellaOps.Feedser.Core.csproj b/src/StellaOps.Feedser.Core/StellaOps.Feedser.Core.csproj new file mode 100644 index 00000000..c74e9e04 --- /dev/null +++ b/src/StellaOps.Feedser.Core/StellaOps.Feedser.Core.csproj @@ -0,0 +1,19 @@ + + + net10.0 + preview + enable + enable + true + + + + + + + + + + + + diff --git a/src/StellaOps.Feedser.Core/TASKS.md b/src/StellaOps.Feedser.Core/TASKS.md new file mode 100644 index 00000000..a2dfb2cc --- /dev/null +++ b/src/StellaOps.Feedser.Core/TASKS.md @@ -0,0 +1,16 @@ +# TASKS +| Task | Owner(s) | Depends on | Notes | +|---|---|---|---| +|JobCoordinator implementation (create/get/mark status)|BE-Core|Storage.Mongo|DONE – `JobCoordinator` drives Mongo-backed runs.| +|Cron scheduling loop with TimeProvider|BE-Core|Core|DONE – `JobSchedulerHostedService` evaluates cron expressions.| +|Single-flight/lease semantics|BE-Core|Storage.Mongo|DONE – lease acquisition backed by `MongoLeaseStore`.| +|Trigger API contract (Result mapping)|BE-Core|WebService|DONE – `JobTriggerResult` outcomes map to HTTP statuses.| +|Run telemetry enrichment|BE-Core|Observability|DONE – `JobDiagnostics` ties activities & counters into coordinator/scheduler paths.| +|Deterministic params hashing|BE-Core|Core|DONE – `JobParametersHasher` creates SHA256 hash.| +|Golden tests for timeout/cancel|QA|Core|DONE – JobCoordinatorTests cover cancellation timeout path.| +|JobSchedulerBuilder options registry coverage|BE-Core|Core|DONE – added scheduler tests confirming cron/timeout/lease metadata persists via JobSchedulerOptions.| +|Plugin discovery + DI glue with PluginHost|BE-Core|Plugin libs|DONE – JobPluginRegistrationExtensions now loads PluginHost routines and wires connector/exporter registrations.| +|Harden lease release error handling in JobCoordinator|BE-Core|Storage.Mongo|DONE – lease release failures now logged, wrapped, and drive run failure status; fire-and-forget execution guarded. Verified with `dotnet test --no-build --filter JobCoordinator`.| +|Validate job trigger parameters for serialization|BE-Core|WebService|DONE – trigger parameters normalized/serialized with defensive checks returning InvalidParameters on failure. Full-suite `dotnet test --no-build` currently red from live connector fixture drift (Oracle/JVN/RedHat).| +|FEEDCORE-ENGINE-03-001 Canonical merger implementation|BE-Core|Merge|DONE – `CanonicalMerger` applies GHSA/NVD/OSV conflict rules with deterministic provenance and comprehensive unit coverage. **Coordination:** Connector leads must align mapper outputs with the canonical field expectations before 2025-10-18 so Merge can activate the path globally.| +|FEEDCORE-ENGINE-03-002 Field precedence and tie-breaker map|BE-Core|Merge|DONE – field precedence and freshness overrides enforced via `FieldPrecedence` map with tie-breakers and analytics capture. **Reminder:** Storage/Merge owners review precedence overrides when onboarding new feeds to ensure `decisionReason` tagging stays consistent.| diff --git a/src/StellaOps.Feedser.Exporter.Json.Tests/JsonExportSnapshotBuilderTests.cs b/src/StellaOps.Feedser.Exporter.Json.Tests/JsonExportSnapshotBuilderTests.cs new file mode 100644 index 00000000..4dada488 --- /dev/null +++ b/src/StellaOps.Feedser.Exporter.Json.Tests/JsonExportSnapshotBuilderTests.cs @@ -0,0 +1,213 @@ +using System; +using System.Collections.Generic; +using System.Globalization; +using System.IO; +using System.Linq; +using System.Security.Cryptography; +using System.Runtime.CompilerServices; +using System.Threading; +using System.Threading.Tasks; +using StellaOps.Feedser.Exporter.Json; +using StellaOps.Feedser.Models; + +namespace StellaOps.Feedser.Exporter.Json.Tests; + +public sealed class JsonExportSnapshotBuilderTests : IDisposable +{ + private readonly string _root; + + public JsonExportSnapshotBuilderTests() + { + _root = Directory.CreateTempSubdirectory("feedser-json-export-tests").FullName; + } + + [Fact] + public async Task WritesDeterministicTree() + { + var options = new JsonExportOptions { OutputRoot = _root }; + var builder = new JsonExportSnapshotBuilder(options, new VulnListJsonExportPathResolver()); + var exportedAt = DateTimeOffset.Parse("2024-07-15T12:00:00Z", CultureInfo.InvariantCulture); + + var advisories = new[] + { + CreateAdvisory( + advisoryKey: "CVE-2024-9999", + aliases: new[] { "GHSA-zzzz-yyyy-xxxx", "CVE-2024-9999" }, + title: "Deterministic Snapshot", + severity: "critical"), + CreateAdvisory( + advisoryKey: "VENDOR-2024-42", + aliases: new[] { "ALIAS-1", "ALIAS-2" }, + title: "Vendor Advisory", + severity: "medium"), + }; + + var result = await builder.WriteAsync(advisories, exportedAt, cancellationToken: CancellationToken.None); + + Assert.Equal(advisories.Length, result.AdvisoryCount); + Assert.Equal(exportedAt, result.ExportedAt); + + var expectedFiles = result.FilePaths.OrderBy(x => x, StringComparer.Ordinal).ToArray(); + Assert.Contains("nvd/2024/CVE-2024-9999.json", expectedFiles); + Assert.Contains("misc/VENDOR-2024-42.json", expectedFiles); + + var cvePath = ResolvePath(result.ExportDirectory, "nvd/2024/CVE-2024-9999.json"); + Assert.True(File.Exists(cvePath)); + var actualJson = await File.ReadAllTextAsync(cvePath, CancellationToken.None); + Assert.Equal(SnapshotSerializer.ToSnapshot(advisories[0]), actualJson); + } + + [Fact] + public async Task ProducesIdenticalBytesAcrossRuns() + { + var options = new JsonExportOptions { OutputRoot = _root }; + var builder = new JsonExportSnapshotBuilder(options, new VulnListJsonExportPathResolver()); + var exportedAt = DateTimeOffset.Parse("2024-05-01T00:00:00Z", CultureInfo.InvariantCulture); + var advisories = new[] + { + CreateAdvisory("CVE-2024-1000", new[] { "CVE-2024-1000", "GHSA-aaaa-bbbb-cccc" }, "Snapshot Stable", "high"), + }; + + var first = await builder.WriteAsync(advisories, exportedAt, exportName: "20240501T000000Z", CancellationToken.None); + var firstDigest = ComputeDigest(first); + + var second = await builder.WriteAsync(advisories, exportedAt, exportName: "20240501T000000Z", CancellationToken.None); + var secondDigest = ComputeDigest(second); + + Assert.Equal(Convert.ToHexString(firstDigest), Convert.ToHexString(secondDigest)); + } + + [Fact] + public async Task WriteAsync_NormalizesInputOrdering() + { + var options = new JsonExportOptions { OutputRoot = _root }; + var builder = new JsonExportSnapshotBuilder(options, new VulnListJsonExportPathResolver()); + var exportedAt = DateTimeOffset.Parse("2024-06-01T00:00:00Z", CultureInfo.InvariantCulture); + + var advisoryA = CreateAdvisory("CVE-2024-1000", new[] { "CVE-2024-1000" }, "Alpha", "high"); + var advisoryB = CreateAdvisory("VENDOR-0001", new[] { "VENDOR-0001" }, "Vendor Advisory", "medium"); + + var result = await builder.WriteAsync(new[] { advisoryB, advisoryA }, exportedAt, cancellationToken: CancellationToken.None); + + var expectedOrder = result.FilePaths.OrderBy(path => path, StringComparer.Ordinal).ToArray(); + Assert.Equal(expectedOrder, result.FilePaths.ToArray()); + } + + [Fact] + public async Task WriteAsync_EnumeratesStreamOnlyOnce() + { + var options = new JsonExportOptions { OutputRoot = _root }; + var builder = new JsonExportSnapshotBuilder(options, new VulnListJsonExportPathResolver()); + var exportedAt = DateTimeOffset.Parse("2024-08-01T00:00:00Z", CultureInfo.InvariantCulture); + + var advisories = new[] + { + CreateAdvisory("CVE-2024-2000", new[] { "CVE-2024-2000" }, "Streaming One", "medium"), + CreateAdvisory("CVE-2024-2001", new[] { "CVE-2024-2001" }, "Streaming Two", "low"), + }; + + var sequence = new SingleEnumerationAsyncSequence(advisories); + var result = await builder.WriteAsync(sequence, exportedAt, cancellationToken: CancellationToken.None); + + Assert.Equal(advisories.Length, result.AdvisoryCount); + } + + private static Advisory CreateAdvisory(string advisoryKey, string[] aliases, string title, string severity) + { + return new Advisory( + advisoryKey: advisoryKey, + title: title, + summary: null, + language: "EN", + published: DateTimeOffset.Parse("2024-01-01T00:00:00Z", CultureInfo.InvariantCulture), + modified: DateTimeOffset.Parse("2024-01-02T00:00:00Z", CultureInfo.InvariantCulture), + severity: severity, + exploitKnown: false, + aliases: aliases, + references: new[] + { + new AdvisoryReference("https://example.com/advisory", "advisory", null, null, AdvisoryProvenance.Empty), + }, + affectedPackages: new[] + { + new AffectedPackage( + AffectedPackageTypes.SemVer, + "sample/package", + platform: null, + versionRanges: Array.Empty(), + statuses: Array.Empty(), + provenance: Array.Empty()), + }, + cvssMetrics: Array.Empty(), + provenance: new[] + { + new AdvisoryProvenance("feedser", "normalized", "canonical", DateTimeOffset.Parse("2024-01-02T00:00:00Z", CultureInfo.InvariantCulture)), + }); + } + + private static byte[] ComputeDigest(JsonExportResult result) + { + using var sha256 = SHA256.Create(); + foreach (var relative in result.FilePaths.OrderBy(x => x, StringComparer.Ordinal)) + { + var fullPath = ResolvePath(result.ExportDirectory, relative); + var bytes = File.ReadAllBytes(fullPath); + sha256.TransformBlock(bytes, 0, bytes.Length, null, 0); + } + + sha256.TransformFinalBlock(Array.Empty(), 0, 0); + return sha256.Hash ?? Array.Empty(); + } + + private static string ResolvePath(string root, string relative) + { + var segments = relative.Split('/', StringSplitOptions.RemoveEmptyEntries); + return Path.Combine(new[] { root }.Concat(segments).ToArray()); + } + + public void Dispose() + { + try + { + if (Directory.Exists(_root)) + { + Directory.Delete(_root, recursive: true); + } + } + catch + { + // best effort cleanup + } + } + + private sealed class SingleEnumerationAsyncSequence : IAsyncEnumerable + { + private readonly IReadOnlyList _advisories; + private int _enumerated; + + public SingleEnumerationAsyncSequence(IReadOnlyList advisories) + { + _advisories = advisories ?? throw new ArgumentNullException(nameof(advisories)); + } + + public IAsyncEnumerator GetAsyncEnumerator(CancellationToken cancellationToken = default) + { + if (Interlocked.Exchange(ref _enumerated, 1) == 1) + { + throw new InvalidOperationException("Sequence was enumerated more than once."); + } + + return Enumerate(cancellationToken); + + async IAsyncEnumerator Enumerate([EnumeratorCancellation] CancellationToken ct) + { + foreach (var advisory in _advisories) + { + ct.ThrowIfCancellationRequested(); + yield return advisory; + await Task.Yield(); + } + } + } + } +} diff --git a/src/StellaOps.Feedser.Exporter.Json.Tests/JsonExporterDependencyInjectionRoutineTests.cs b/src/StellaOps.Feedser.Exporter.Json.Tests/JsonExporterDependencyInjectionRoutineTests.cs new file mode 100644 index 00000000..9521557f --- /dev/null +++ b/src/StellaOps.Feedser.Exporter.Json.Tests/JsonExporterDependencyInjectionRoutineTests.cs @@ -0,0 +1,83 @@ +using System.Collections.Generic; +using System.Runtime.CompilerServices; +using System.Threading.Tasks; +using Microsoft.Extensions.Configuration; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Options; +using StellaOps.Feedser.Core.Jobs; +using StellaOps.Feedser.Exporter.Json; +using StellaOps.Feedser.Storage.Mongo.Advisories; +using StellaOps.Feedser.Storage.Mongo.Exporting; +using StellaOps.Feedser.Models; + +namespace StellaOps.Feedser.Exporter.Json.Tests; + +public sealed class JsonExporterDependencyInjectionRoutineTests +{ + [Fact] + public void Register_AddsJobDefinitionAndServices() + { + var services = new ServiceCollection(); + services.AddLogging(); + services.AddSingleton(); + services.AddSingleton(); + services.AddOptions(); + + var configuration = new ConfigurationBuilder() + .AddInMemoryCollection(new Dictionary()) + .Build(); + + var routine = new JsonExporterDependencyInjectionRoutine(); + routine.Register(services, configuration); + + using var provider = services.BuildServiceProvider(); + var optionsAccessor = provider.GetRequiredService>(); + var options = optionsAccessor.Value; + + Assert.True(options.Definitions.TryGetValue(JsonExportJob.JobKind, out var definition)); + Assert.Equal(typeof(JsonExportJob), definition.JobType); + Assert.True(definition.Enabled); + + var exporter = provider.GetRequiredService(); + Assert.NotNull(exporter); + } + + private sealed class StubAdvisoryStore : IAdvisoryStore + { + public Task> GetRecentAsync(int limit, CancellationToken cancellationToken) + => Task.FromResult>(Array.Empty()); + + public Task FindAsync(string advisoryKey, CancellationToken cancellationToken) + => Task.FromResult(null); + + public Task UpsertAsync(Advisory advisory, CancellationToken cancellationToken) + => Task.CompletedTask; + + public IAsyncEnumerable StreamAsync(CancellationToken cancellationToken) + { + return Enumerate(cancellationToken); + + static async IAsyncEnumerable Enumerate([EnumeratorCancellation] CancellationToken ct) + { + ct.ThrowIfCancellationRequested(); + await Task.Yield(); + yield break; + } + } + } + + private sealed class StubExportStateStore : IExportStateStore + { + private ExportStateRecord? _record; + + public Task FindAsync(string id, CancellationToken cancellationToken) + => Task.FromResult(_record); + + public Task UpsertAsync(ExportStateRecord record, CancellationToken cancellationToken) + { + _record = record; + return Task.FromResult(record); + } + } +} diff --git a/src/StellaOps.Feedser.Exporter.Json.Tests/JsonExporterParitySmokeTests.cs b/src/StellaOps.Feedser.Exporter.Json.Tests/JsonExporterParitySmokeTests.cs new file mode 100644 index 00000000..49aca086 --- /dev/null +++ b/src/StellaOps.Feedser.Exporter.Json.Tests/JsonExporterParitySmokeTests.cs @@ -0,0 +1,182 @@ +using System; +using System.Collections.Generic; +using System.Globalization; +using System.IO; +using System.Linq; +using System.Threading; +using System.Threading.Tasks; +using StellaOps.Feedser.Exporter.Json; +using StellaOps.Feedser.Models; + +namespace StellaOps.Feedser.Exporter.Json.Tests; + +public sealed class JsonExporterParitySmokeTests : IDisposable +{ + private readonly string _root; + + public JsonExporterParitySmokeTests() + { + _root = Directory.CreateTempSubdirectory("feedser-json-parity-tests").FullName; + } + + [Fact] + public async Task ExportProducesVulnListCompatiblePaths() + { + var options = new JsonExportOptions { OutputRoot = _root }; + var builder = new JsonExportSnapshotBuilder(options, new VulnListJsonExportPathResolver()); + var exportedAt = DateTimeOffset.Parse("2024-09-01T12:00:00Z", CultureInfo.InvariantCulture); + + var advisories = CreateSampleAdvisories(); + var result = await builder.WriteAsync(advisories, exportedAt, exportName: "parity-test", CancellationToken.None); + + var expected = new[] + { + "amazon/2/ALAS2-2024-1234.json", + "debian/DLA-2024-1234.json", + "ghsa/go/github.com%2Facme%2Fsample/GHSA-AAAA-BBBB-CCCC.json", + "nvd/2023/CVE-2023-27524.json", + "oracle/linux/ELSA-2024-12345.json", + "redhat/oval/RHSA-2024_0252.json", + "ubuntu/USN-6620-1.json", + "wolfi/WOLFI-2024-0001.json", + }; + + Assert.Equal(expected, result.FilePaths.ToArray()); + + foreach (var path in expected) + { + var fullPath = ResolvePath(result.ExportDirectory, path); + Assert.True(File.Exists(fullPath), $"Expected export file '{path}' to be present"); + } + } + + private static IReadOnlyList CreateSampleAdvisories() + { + var published = DateTimeOffset.Parse("2024-01-01T00:00:00Z", CultureInfo.InvariantCulture); + var modified = DateTimeOffset.Parse("2024-02-01T00:00:00Z", CultureInfo.InvariantCulture); + + return new[] + { + CreateAdvisory( + "CVE-2023-27524", + "Apache Superset Improper Authentication", + new[] { "CVE-2023-27524" }, + null, + "nvd", + published, + modified), + CreateAdvisory( + "GHSA-aaaa-bbbb-cccc", + "Sample GHSA", + new[] { "CVE-2024-2000" }, + new[] + { + new AffectedPackage( + AffectedPackageTypes.SemVer, + "pkg:go/github.com/acme/sample@1.0.0", + provenance: new[] { new AdvisoryProvenance("ghsa", "map", "", published) }) + }, + "ghsa", + published, + modified), + CreateAdvisory( + "USN-6620-1", + "Ubuntu Security Notice", + null, + null, + "ubuntu", + published, + modified), + CreateAdvisory( + "DLA-2024-1234", + "Debian LTS Advisory", + null, + null, + "debian", + published, + modified), + CreateAdvisory( + "RHSA-2024:0252", + "Red Hat Security Advisory", + null, + null, + "redhat", + published, + modified), + CreateAdvisory( + "ALAS2-2024-1234", + "Amazon Linux Advisory", + null, + null, + "amazon", + published, + modified), + CreateAdvisory( + "ELSA-2024-12345", + "Oracle Linux Advisory", + null, + null, + "oracle", + published, + modified), + CreateAdvisory( + "WOLFI-2024-0001", + "Wolfi Advisory", + null, + null, + "wolfi", + published, + modified), + }; + } + + private static Advisory CreateAdvisory( + string advisoryKey, + string title, + IEnumerable? aliases, + IEnumerable? packages, + string? provenanceSource, + DateTimeOffset? published, + DateTimeOffset? modified) + { + var provenance = provenanceSource is null + ? Array.Empty() + : new[] { new AdvisoryProvenance(provenanceSource, "normalize", "", modified ?? DateTimeOffset.UtcNow) }; + + return new Advisory( + advisoryKey, + title, + summary: null, + language: "en", + published, + modified, + severity: "medium", + exploitKnown: false, + aliases: aliases ?? Array.Empty(), + references: Array.Empty(), + affectedPackages: packages ?? Array.Empty(), + cvssMetrics: Array.Empty(), + provenance: provenance); + } + + private static string ResolvePath(string root, string relative) + { + var segments = relative.Split('/', StringSplitOptions.RemoveEmptyEntries); + return Path.Combine(new[] { root }.Concat(segments).ToArray()); + } + + public void Dispose() + { + try + { + if (Directory.Exists(_root)) + { + Directory.Delete(_root, recursive: true); + } + } + catch + { + // best effort cleanup + } + } +} diff --git a/src/StellaOps.Feedser.Exporter.Json.Tests/JsonFeedExporterTests.cs b/src/StellaOps.Feedser.Exporter.Json.Tests/JsonFeedExporterTests.cs new file mode 100644 index 00000000..f278c226 --- /dev/null +++ b/src/StellaOps.Feedser.Exporter.Json.Tests/JsonFeedExporterTests.cs @@ -0,0 +1,265 @@ +using System; +using System.Collections.Generic; +using System.Globalization; +using System.IO; +using System.Linq; +using System.Runtime.CompilerServices; +using System.Text.Json; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Logging.Abstractions; +using Microsoft.Extensions.Options; +using StellaOps.Feedser.Exporter.Json; +using StellaOps.Feedser.Models; +using StellaOps.Feedser.Storage.Mongo.Advisories; +using StellaOps.Feedser.Storage.Mongo.Exporting; + +namespace StellaOps.Feedser.Exporter.Json.Tests; + +public sealed class JsonFeedExporterTests : IDisposable +{ + private readonly string _root; + + public JsonFeedExporterTests() + { + _root = Directory.CreateTempSubdirectory("feedser-json-exporter-tests").FullName; + } + + [Fact] + public async Task ExportAsync_SkipsWhenDigestUnchanged() + { + var advisory = new Advisory( + advisoryKey: "CVE-2024-1234", + title: "Test Advisory", + summary: null, + language: "en", + published: DateTimeOffset.Parse("2024-01-01T00:00:00Z", CultureInfo.InvariantCulture), + modified: DateTimeOffset.Parse("2024-01-02T00:00:00Z", CultureInfo.InvariantCulture), + severity: "high", + exploitKnown: false, + aliases: new[] { "CVE-2024-1234" }, + references: Array.Empty(), + affectedPackages: Array.Empty(), + cvssMetrics: Array.Empty(), + provenance: Array.Empty()); + + var advisoryStore = new StubAdvisoryStore(advisory); + var options = Options.Create(new JsonExportOptions + { + OutputRoot = _root, + MaintainLatestSymlink = false, + }); + + var stateStore = new InMemoryExportStateStore(); + var timeProvider = new TestTimeProvider(DateTimeOffset.Parse("2024-07-15T12:00:00Z", CultureInfo.InvariantCulture)); + var stateManager = new ExportStateManager(stateStore, timeProvider); + var exporter = new JsonFeedExporter( + advisoryStore, + options, + new VulnListJsonExportPathResolver(), + stateManager, + NullLogger.Instance, + timeProvider); + + using var provider = new ServiceCollection().BuildServiceProvider(); + await exporter.ExportAsync(provider, CancellationToken.None); + + var record = await stateStore.FindAsync(JsonFeedExporter.ExporterId, CancellationToken.None); + Assert.NotNull(record); + var firstUpdated = record!.UpdatedAt; + Assert.Equal("20240715T120000Z", record.BaseExportId); + Assert.Equal(record.LastFullDigest, record.ExportCursor); + + var firstExportPath = Path.Combine(_root, "20240715T120000Z"); + Assert.True(Directory.Exists(firstExportPath)); + + timeProvider.Advance(TimeSpan.FromMinutes(5)); + await exporter.ExportAsync(provider, CancellationToken.None); + + record = await stateStore.FindAsync(JsonFeedExporter.ExporterId, CancellationToken.None); + Assert.NotNull(record); + Assert.Equal(firstUpdated, record!.UpdatedAt); + + var secondExportPath = Path.Combine(_root, "20240715T120500Z"); + Assert.False(Directory.Exists(secondExportPath)); + } + + [Fact] + public async Task ExportAsync_WritesManifestMetadata() + { + var exportedAt = DateTimeOffset.Parse("2024-08-10T00:00:00Z", CultureInfo.InvariantCulture); + var advisory = new Advisory( + advisoryKey: "CVE-2024-4321", + title: "Manifest Test", + summary: null, + language: "en", + published: DateTimeOffset.Parse("2024-07-01T00:00:00Z", CultureInfo.InvariantCulture), + modified: DateTimeOffset.Parse("2024-07-02T00:00:00Z", CultureInfo.InvariantCulture), + severity: "medium", + exploitKnown: false, + aliases: new[] { "CVE-2024-4321" }, + references: Array.Empty(), + affectedPackages: Array.Empty(), + cvssMetrics: Array.Empty(), + provenance: Array.Empty()); + + var advisoryStore = new StubAdvisoryStore(advisory); + var optionsValue = new JsonExportOptions + { + OutputRoot = _root, + MaintainLatestSymlink = false, + }; + + var options = Options.Create(optionsValue); + var stateStore = new InMemoryExportStateStore(); + var timeProvider = new TestTimeProvider(exportedAt); + var stateManager = new ExportStateManager(stateStore, timeProvider); + var exporter = new JsonFeedExporter( + advisoryStore, + options, + new VulnListJsonExportPathResolver(), + stateManager, + NullLogger.Instance, + timeProvider); + + using var provider = new ServiceCollection().BuildServiceProvider(); + await exporter.ExportAsync(provider, CancellationToken.None); + + var exportId = exportedAt.ToString(optionsValue.DirectoryNameFormat, CultureInfo.InvariantCulture); + var exportDirectory = Path.Combine(_root, exportId); + var manifestPath = Path.Combine(exportDirectory, "manifest.json"); + + Assert.True(File.Exists(manifestPath)); + + using var document = JsonDocument.Parse(await File.ReadAllBytesAsync(manifestPath, CancellationToken.None)); + var root = document.RootElement; + + Assert.Equal(exportId, root.GetProperty("exportId").GetString()); + Assert.Equal(exportedAt.UtcDateTime, root.GetProperty("generatedAt").GetDateTime()); + Assert.Equal(1, root.GetProperty("advisoryCount").GetInt32()); + + var exportedFiles = Directory.EnumerateFiles(exportDirectory, "*.json", SearchOption.AllDirectories) + .Select(path => new + { + Absolute = path, + Relative = Path.GetRelativePath(exportDirectory, path).Replace("\\", "/", StringComparison.Ordinal), + }) + .Where(file => !string.Equals(file.Relative, "manifest.json", StringComparison.OrdinalIgnoreCase)) + .OrderBy(file => file.Relative, StringComparer.Ordinal) + .ToArray(); + + var filesElement = root.GetProperty("files") + .EnumerateArray() + .Select(element => new + { + Path = element.GetProperty("path").GetString(), + Bytes = element.GetProperty("bytes").GetInt64(), + Digest = element.GetProperty("digest").GetString(), + }) + .OrderBy(file => file.Path, StringComparer.Ordinal) + .ToArray(); + + Assert.Equal(exportedFiles.Select(file => file.Relative).ToArray(), filesElement.Select(file => file.Path).ToArray()); + + long totalBytes = exportedFiles.Select(file => new FileInfo(file.Absolute).Length).Sum(); + Assert.Equal(totalBytes, root.GetProperty("totalBytes").GetInt64()); + Assert.Equal(exportedFiles.Length, root.GetProperty("fileCount").GetInt32()); + + var digest = root.GetProperty("digest").GetString(); + var digestResult = new JsonExportResult( + exportDirectory, + exportedAt, + exportedFiles.Select(file => + { + var manifestEntry = filesElement.First(f => f.Path == file.Relative); + if (manifestEntry.Digest is null) + { + throw new InvalidOperationException($"Manifest entry for {file.Relative} missing digest."); + } + + return new JsonExportFile(file.Relative, new FileInfo(file.Absolute).Length, manifestEntry.Digest); + }), + exportedFiles.Length, + totalBytes); + var expectedDigest = ExportDigestCalculator.ComputeTreeDigest(digestResult); + Assert.Equal(expectedDigest, digest); + + var exporterVersion = root.GetProperty("exporterVersion").GetString(); + Assert.Equal(ExporterVersion.GetVersion(typeof(JsonFeedExporter)), exporterVersion); + } + + public void Dispose() + { + try + { + if (Directory.Exists(_root)) + { + Directory.Delete(_root, recursive: true); + } + } + catch + { + // best effort cleanup + } + } + + private sealed class StubAdvisoryStore : IAdvisoryStore + { + private readonly IReadOnlyList _advisories; + + public StubAdvisoryStore(params Advisory[] advisories) + { + _advisories = advisories; + } + + public Task> GetRecentAsync(int limit, CancellationToken cancellationToken) + => Task.FromResult(_advisories); + + public Task FindAsync(string advisoryKey, CancellationToken cancellationToken) + => Task.FromResult(_advisories.FirstOrDefault(a => a.AdvisoryKey == advisoryKey)); + + public Task UpsertAsync(Advisory advisory, CancellationToken cancellationToken) + => Task.CompletedTask; + + public IAsyncEnumerable StreamAsync(CancellationToken cancellationToken) + { + return EnumerateAsync(cancellationToken); + + async IAsyncEnumerable EnumerateAsync([EnumeratorCancellation] CancellationToken ct) + { + foreach (var advisory in _advisories) + { + ct.ThrowIfCancellationRequested(); + yield return advisory; + await Task.Yield(); + } + } + } + } + + private sealed class InMemoryExportStateStore : IExportStateStore + { + private ExportStateRecord? _record; + + public Task FindAsync(string id, CancellationToken cancellationToken) + => Task.FromResult(_record); + + public Task UpsertAsync(ExportStateRecord record, CancellationToken cancellationToken) + { + _record = record; + return Task.FromResult(record); + } + } + + private sealed class TestTimeProvider : TimeProvider + { + private DateTimeOffset _now; + + public TestTimeProvider(DateTimeOffset start) => _now = start; + + public override DateTimeOffset GetUtcNow() => _now; + + public void Advance(TimeSpan delta) => _now = _now.Add(delta); + } +} diff --git a/src/StellaOps.Feedser.Exporter.Json.Tests/StellaOps.Feedser.Exporter.Json.Tests.csproj b/src/StellaOps.Feedser.Exporter.Json.Tests/StellaOps.Feedser.Exporter.Json.Tests.csproj new file mode 100644 index 00000000..e579ef8c --- /dev/null +++ b/src/StellaOps.Feedser.Exporter.Json.Tests/StellaOps.Feedser.Exporter.Json.Tests.csproj @@ -0,0 +1,13 @@ + + + net10.0 + enable + enable + + + + + + + + diff --git a/src/StellaOps.Feedser.Exporter.Json.Tests/VulnListJsonExportPathResolverTests.cs b/src/StellaOps.Feedser.Exporter.Json.Tests/VulnListJsonExportPathResolverTests.cs new file mode 100644 index 00000000..8085a96d --- /dev/null +++ b/src/StellaOps.Feedser.Exporter.Json.Tests/VulnListJsonExportPathResolverTests.cs @@ -0,0 +1,148 @@ +using System.Collections.Generic; +using System.Globalization; +using System.IO; +using StellaOps.Feedser.Exporter.Json; +using StellaOps.Feedser.Models; + +namespace StellaOps.Feedser.Exporter.Json.Tests; + +public sealed class VulnListJsonExportPathResolverTests +{ + private static readonly DateTimeOffset DefaultPublished = DateTimeOffset.Parse("2024-01-01T00:00:00Z", CultureInfo.InvariantCulture); + + [Fact] + public void ResolvesCvePath() + { + var advisory = CreateAdvisory("CVE-2024-1234"); + var resolver = new VulnListJsonExportPathResolver(); + + var path = resolver.GetRelativePath(advisory); + + Assert.Equal(Path.Combine("nvd", "2024", "CVE-2024-1234.json"), path); + } + + [Fact] + public void ResolvesGhsaWithPackage() + { + var package = new AffectedPackage( + AffectedPackageTypes.SemVer, + "pkg:go/github.com/acme/widget@1.0.0", + platform: null, + versionRanges: Array.Empty(), + statuses: Array.Empty(), + provenance: Array.Empty()); + + var advisory = CreateAdvisory( + "GHSA-aaaa-bbbb-cccc", + aliases: new[] { "CVE-2024-2000" }, + packages: new[] { package }); + var resolver = new VulnListJsonExportPathResolver(); + + var path = resolver.GetRelativePath(advisory); + + Assert.Equal(Path.Combine("ghsa", "go", "github.com%2Facme%2Fwidget", "GHSA-AAAA-BBBB-CCCC.json"), path); + } + + [Fact] + public void ResolvesUbuntuUsn() + { + var advisory = CreateAdvisory("USN-6620-1"); + var resolver = new VulnListJsonExportPathResolver(); + var path = resolver.GetRelativePath(advisory); + + Assert.Equal(Path.Combine("ubuntu", "USN-6620-1.json"), path); + } + + [Fact] + public void ResolvesDebianDla() + { + var advisory = CreateAdvisory("DLA-1234-1"); + var resolver = new VulnListJsonExportPathResolver(); + var path = resolver.GetRelativePath(advisory); + + Assert.Equal(Path.Combine("debian", "DLA-1234-1.json"), path); + } + + [Fact] + public void ResolvesRedHatRhsa() + { + var advisory = CreateAdvisory("RHSA-2024:0252"); + var resolver = new VulnListJsonExportPathResolver(); + var path = resolver.GetRelativePath(advisory); + + Assert.Equal(Path.Combine("redhat", "oval", "RHSA-2024_0252.json"), path); + } + + [Fact] + public void ResolvesAmazonAlas() + { + var advisory = CreateAdvisory("ALAS2-2024-1234"); + var resolver = new VulnListJsonExportPathResolver(); + var path = resolver.GetRelativePath(advisory); + + Assert.Equal(Path.Combine("amazon", "2", "ALAS2-2024-1234.json"), path); + } + + [Fact] + public void ResolvesOracleElsa() + { + var advisory = CreateAdvisory("ELSA-2024-12345"); + var resolver = new VulnListJsonExportPathResolver(); + var path = resolver.GetRelativePath(advisory); + + Assert.Equal(Path.Combine("oracle", "linux", "ELSA-2024-12345.json"), path); + } + + [Fact] + public void ResolvesRockyRlsa() + { + var advisory = CreateAdvisory("RLSA-2024:0417"); + var resolver = new VulnListJsonExportPathResolver(); + var path = resolver.GetRelativePath(advisory); + + Assert.Equal(Path.Combine("rocky", "RLSA-2024_0417.json"), path); + } + + [Fact] + public void ResolvesByProvenanceFallback() + { + var provenance = new[] { new AdvisoryProvenance("wolfi", "map", "", DefaultPublished) }; + var advisory = CreateAdvisory("WOLFI-2024-0001", provenance: provenance); + var resolver = new VulnListJsonExportPathResolver(); + var path = resolver.GetRelativePath(advisory); + + Assert.Equal(Path.Combine("wolfi", "WOLFI-2024-0001.json"), path); + } + + [Fact] + public void DefaultsToMiscWhenUnmapped() + { + var advisory = CreateAdvisory("CUSTOM-2024-99"); + var resolver = new VulnListJsonExportPathResolver(); + var path = resolver.GetRelativePath(advisory); + + Assert.Equal(Path.Combine("misc", "CUSTOM-2024-99.json"), path); + } + + private static Advisory CreateAdvisory( + string advisoryKey, + IEnumerable? aliases = null, + IEnumerable? packages = null, + IEnumerable? provenance = null) + { + return new Advisory( + advisoryKey: advisoryKey, + title: $"Advisory {advisoryKey}", + summary: null, + language: "en", + published: DefaultPublished, + modified: DefaultPublished, + severity: "medium", + exploitKnown: false, + aliases: aliases ?? Array.Empty(), + references: Array.Empty(), + affectedPackages: packages ?? Array.Empty(), + cvssMetrics: Array.Empty(), + provenance: provenance ?? Array.Empty()); + } +} diff --git a/src/StellaOps.Feedser.Exporter.Json/AGENTS.md b/src/StellaOps.Feedser.Exporter.Json/AGENTS.md new file mode 100644 index 00000000..25046404 --- /dev/null +++ b/src/StellaOps.Feedser.Exporter.Json/AGENTS.md @@ -0,0 +1,28 @@ +# AGENTS +## Role +Optional exporter producing vuln-list-shaped JSON tree for downstream trivy-db builder or interoperability. Deterministic, provenance-preserving. +## Scope +- Transform canonical advisories into directory tree structure mirroring aquasecurity/vuln-list (by ecosystem/vendor/distro as applicable). +- Sorting and serialization invariants: stable key order, newline policy, UTC ISO-8601. +- Cursoring/incremental export: export_state tracks last advisory hash/time to avoid full rewrites. +- Packaging: output directory under exports/json/ with reproducible naming; optionally symlink latest. +- Optional auxiliary index files (for example severity summaries) may be generated when explicitly requested, but must remain deterministic and avoid altering canonical payloads. +## Participants +- Storage.Mongo.AdvisoryStore as input; ExportState repository for cursors/digests. +- Core scheduler runs JsonExportJob; Plugin DI wires JsonExporter + job. +- TrivyDb exporter may consume the rendered tree in v0 (builder path) if configured. +## Interfaces & contracts +- Job kind: export:json (JsonExportJob). +- Determinism: same inputs -> identical file bytes; hash snapshot persisted. +- Provenance: include minimal provenance fields when helpful; keep identity stable. +## In/Out of scope +In: JSON rendering and layout; incremental/deterministic writes. +Out: ORAS push and Trivy DB BoltDB writing (owned by Trivy exporter). +## Observability & security expectations +- Metrics: export.json.records, bytes, duration, delta.changed. +- Logs: target path, record counts, digest; no sensitive data. +## Tests +- Author and review coverage in `../StellaOps.Feedser.Exporter.Json.Tests`. +- Shared fixtures (e.g., `MongoIntegrationFixture`, `ConnectorTestHarness`) live in `../StellaOps.Feedser.Testing`. +- Keep fixtures deterministic; match new cases to real-world advisories or regression scenarios. + diff --git a/src/StellaOps.Feedser.Exporter.Json/ExportDigestCalculator.cs b/src/StellaOps.Feedser.Exporter.Json/ExportDigestCalculator.cs new file mode 100644 index 00000000..64079258 --- /dev/null +++ b/src/StellaOps.Feedser.Exporter.Json/ExportDigestCalculator.cs @@ -0,0 +1,52 @@ +using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using System.Security.Cryptography; +using System.Text; + +namespace StellaOps.Feedser.Exporter.Json; + +public static class ExportDigestCalculator +{ + public static string ComputeTreeDigest(JsonExportResult result) + { + ArgumentNullException.ThrowIfNull(result); + + using var sha256 = SHA256.Create(); + var buffer = new byte[128 * 1024]; + + foreach (var file in result.FilePaths.OrderBy(static path => path, StringComparer.Ordinal)) + { + var normalized = file.Replace("\\", "/"); + var pathBytes = Encoding.UTF8.GetBytes(normalized); + _ = sha256.TransformBlock(pathBytes, 0, pathBytes.Length, null, 0); + + var fullPath = ResolveFullPath(result.ExportDirectory, normalized); + using var stream = File.OpenRead(fullPath); + int read; + while ((read = stream.Read(buffer, 0, buffer.Length)) > 0) + { + _ = sha256.TransformBlock(buffer, 0, read, null, 0); + } + } + + _ = sha256.TransformFinalBlock(Array.Empty(), 0, 0); + var hash = sha256.Hash ?? Array.Empty(); + var hex = Convert.ToHexString(hash).ToLowerInvariant(); + return $"sha256:{hex}"; + } + + private static string ResolveFullPath(string root, string normalizedRelativePath) + { + var segments = normalizedRelativePath.Split('/', StringSplitOptions.RemoveEmptyEntries); + var parts = new string[segments.Length + 1]; + parts[0] = root; + for (var i = 0; i < segments.Length; i++) + { + parts[i + 1] = segments[i]; + } + + return Path.Combine(parts); + } +} diff --git a/src/StellaOps.Feedser.Exporter.Json/ExporterVersion.cs b/src/StellaOps.Feedser.Exporter.Json/ExporterVersion.cs new file mode 100644 index 00000000..351217d2 --- /dev/null +++ b/src/StellaOps.Feedser.Exporter.Json/ExporterVersion.cs @@ -0,0 +1,28 @@ +using System; +using System.Reflection; + +namespace StellaOps.Feedser.Exporter.Json; + +public static class ExporterVersion +{ + public static string GetVersion(Type anchor) + { + ArgumentNullException.ThrowIfNull(anchor); + var assembly = anchor.Assembly; + + var informational = assembly.GetCustomAttribute()?.InformationalVersion; + if (!string.IsNullOrWhiteSpace(informational)) + { + return informational; + } + + var fileVersion = assembly.GetCustomAttribute()?.Version; + if (!string.IsNullOrWhiteSpace(fileVersion)) + { + return fileVersion!; + } + + var version = assembly.GetName().Version; + return version?.ToString() ?? "0.0.0"; + } +} diff --git a/src/StellaOps.Feedser.Exporter.Json/IJsonExportPathResolver.cs b/src/StellaOps.Feedser.Exporter.Json/IJsonExportPathResolver.cs new file mode 100644 index 00000000..e68af8e4 --- /dev/null +++ b/src/StellaOps.Feedser.Exporter.Json/IJsonExportPathResolver.cs @@ -0,0 +1,12 @@ +using StellaOps.Feedser.Models; + +namespace StellaOps.Feedser.Exporter.Json; + +public interface IJsonExportPathResolver +{ + /// + /// Returns the relative path (using platform directory separators) for the supplied advisory. + /// Path must not include the leading export root. + /// + string GetRelativePath(Advisory advisory); +} diff --git a/src/StellaOps.Feedser.Exporter.Json/JsonExportFile.cs b/src/StellaOps.Feedser.Exporter.Json/JsonExportFile.cs new file mode 100644 index 00000000..c57c69a1 --- /dev/null +++ b/src/StellaOps.Feedser.Exporter.Json/JsonExportFile.cs @@ -0,0 +1,37 @@ +using System; + +namespace StellaOps.Feedser.Exporter.Json; + +/// +/// Metadata describing a single file produced by the JSON exporter. +/// +public sealed class JsonExportFile +{ + public JsonExportFile(string relativePath, long length, string digest) + { + RelativePath = relativePath ?? throw new ArgumentNullException(nameof(relativePath)); + if (relativePath.Length == 0) + { + throw new ArgumentException("Relative path cannot be empty.", nameof(relativePath)); + } + + if (length < 0) + { + throw new ArgumentOutOfRangeException(nameof(length)); + } + + Digest = digest ?? throw new ArgumentNullException(nameof(digest)); + if (digest.Length == 0) + { + throw new ArgumentException("Digest cannot be empty.", nameof(digest)); + } + + Length = length; + } + + public string RelativePath { get; } + + public long Length { get; } + + public string Digest { get; } +} diff --git a/src/StellaOps.Feedser.Exporter.Json/JsonExportJob.cs b/src/StellaOps.Feedser.Exporter.Json/JsonExportJob.cs new file mode 100644 index 00000000..904dc421 --- /dev/null +++ b/src/StellaOps.Feedser.Exporter.Json/JsonExportJob.cs @@ -0,0 +1,30 @@ +using System; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.Extensions.Logging; +using StellaOps.Feedser.Core.Jobs; + +namespace StellaOps.Feedser.Exporter.Json; + +public sealed class JsonExportJob : IJob +{ + public const string JobKind = "export:json"; + public static readonly TimeSpan DefaultTimeout = TimeSpan.FromMinutes(10); + public static readonly TimeSpan DefaultLeaseDuration = TimeSpan.FromMinutes(5); + + private readonly JsonFeedExporter _exporter; + private readonly ILogger _logger; + + public JsonExportJob(JsonFeedExporter exporter, ILogger logger) + { + _exporter = exporter ?? throw new ArgumentNullException(nameof(exporter)); + _logger = logger ?? throw new ArgumentNullException(nameof(logger)); + } + + public async Task ExecuteAsync(JobExecutionContext context, CancellationToken cancellationToken) + { + _logger.LogInformation("Executing JSON export job {RunId}", context.RunId); + await _exporter.ExportAsync(context.Services, cancellationToken).ConfigureAwait(false); + _logger.LogInformation("Completed JSON export job {RunId}", context.RunId); + } +} diff --git a/src/StellaOps.Feedser.Exporter.Json/JsonExportManifestWriter.cs b/src/StellaOps.Feedser.Exporter.Json/JsonExportManifestWriter.cs new file mode 100644 index 00000000..7a0c80d7 --- /dev/null +++ b/src/StellaOps.Feedser.Exporter.Json/JsonExportManifestWriter.cs @@ -0,0 +1,66 @@ +using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using System.Text.Json; +using System.Text.Json.Serialization; +using System.Threading; +using System.Threading.Tasks; + +namespace StellaOps.Feedser.Exporter.Json; + +internal static class JsonExportManifestWriter +{ + private static readonly JsonSerializerOptions SerializerOptions = new() + { + PropertyNamingPolicy = JsonNamingPolicy.CamelCase, + DefaultIgnoreCondition = JsonIgnoreCondition.WhenWritingNull, + WriteIndented = true, + }; + + public static async Task WriteAsync( + JsonExportResult result, + string digest, + string exporterVersion, + CancellationToken cancellationToken) + { + ArgumentNullException.ThrowIfNull(result); + ArgumentException.ThrowIfNullOrEmpty(digest); + ArgumentException.ThrowIfNullOrEmpty(exporterVersion); + + var exportId = Path.GetFileName(result.ExportDirectory); + var files = result.Files + .Select(static file => new JsonExportManifestFile(file.RelativePath.Replace("\\", "/", StringComparison.Ordinal), file.Length, file.Digest)) + .ToArray(); + + var manifest = new JsonExportManifest( + exportId, + result.ExportedAt.UtcDateTime, + digest, + result.AdvisoryCount, + result.TotalBytes, + files.Length, + files, + exporterVersion); + + var payload = JsonSerializer.SerializeToUtf8Bytes(manifest, SerializerOptions); + var manifestPath = Path.Combine(result.ExportDirectory, "manifest.json"); + await File.WriteAllBytesAsync(manifestPath, payload, cancellationToken).ConfigureAwait(false); + File.SetLastWriteTimeUtc(manifestPath, result.ExportedAt.UtcDateTime); + } + + private sealed record JsonExportManifest( + [property: JsonPropertyOrder(1)] string ExportId, + [property: JsonPropertyOrder(2)] DateTime GeneratedAt, + [property: JsonPropertyOrder(3)] string Digest, + [property: JsonPropertyOrder(4)] int AdvisoryCount, + [property: JsonPropertyOrder(5)] long TotalBytes, + [property: JsonPropertyOrder(6)] int FileCount, + [property: JsonPropertyOrder(7)] IReadOnlyList Files, + [property: JsonPropertyOrder(8)] string ExporterVersion); + + private sealed record JsonExportManifestFile( + [property: JsonPropertyOrder(1)] string Path, + [property: JsonPropertyOrder(2)] long Bytes, + [property: JsonPropertyOrder(3)] string Digest); +} diff --git a/src/StellaOps.Feedser.Exporter.Json/JsonExportOptions.cs b/src/StellaOps.Feedser.Exporter.Json/JsonExportOptions.cs new file mode 100644 index 00000000..a09cad61 --- /dev/null +++ b/src/StellaOps.Feedser.Exporter.Json/JsonExportOptions.cs @@ -0,0 +1,34 @@ +using System.IO; + +namespace StellaOps.Feedser.Exporter.Json; + +/// +/// Configuration for JSON exporter output paths and determinism controls. +/// +public sealed class JsonExportOptions +{ + /// + /// Root directory where exports are written. Default "exports/json". + /// + public string OutputRoot { get; set; } = Path.Combine("exports", "json"); + + /// + /// Format string applied to the export timestamp to produce the directory name. + /// + public string DirectoryNameFormat { get; set; } = "yyyyMMdd'T'HHmmss'Z'"; + + /// + /// Optional static name for the symlink (or directory junction) pointing at the most recent export. + /// + public string LatestSymlinkName { get; set; } = "latest"; + + /// + /// When true, attempts to re-point after a successful export. + /// + public bool MaintainLatestSymlink { get; set; } = true; + + /// + /// Optional repository identifier recorded alongside export state metadata. + /// + public string? TargetRepository { get; set; } +} diff --git a/src/StellaOps.Feedser.Exporter.Json/JsonExportResult.cs b/src/StellaOps.Feedser.Exporter.Json/JsonExportResult.cs new file mode 100644 index 00000000..0842d1a8 --- /dev/null +++ b/src/StellaOps.Feedser.Exporter.Json/JsonExportResult.cs @@ -0,0 +1,46 @@ +using System; +using System.Collections.Generic; +using System.Collections.Immutable; +using System.Linq; + +namespace StellaOps.Feedser.Exporter.Json; + +public sealed class JsonExportResult +{ + public JsonExportResult( + string exportDirectory, + DateTimeOffset exportedAt, + IEnumerable files, + int advisoryCount, + long totalBytes) + { + if (string.IsNullOrWhiteSpace(exportDirectory)) + { + throw new ArgumentException("Export directory must be provided.", nameof(exportDirectory)); + } + + ExportDirectory = exportDirectory; + ExportedAt = exportedAt; + AdvisoryCount = advisoryCount; + TotalBytes = totalBytes; + + var list = (files ?? throw new ArgumentNullException(nameof(files))) + .Where(static file => file is not null) + .ToImmutableArray(); + + Files = list; + FilePaths = list.Select(static file => file.RelativePath).ToImmutableArray(); + } + + public string ExportDirectory { get; } + + public DateTimeOffset ExportedAt { get; } + + public ImmutableArray Files { get; } + + public ImmutableArray FilePaths { get; } + + public int AdvisoryCount { get; } + + public long TotalBytes { get; } +} diff --git a/src/StellaOps.Feedser.Exporter.Json/JsonExportSnapshotBuilder.cs b/src/StellaOps.Feedser.Exporter.Json/JsonExportSnapshotBuilder.cs new file mode 100644 index 00000000..c622486d --- /dev/null +++ b/src/StellaOps.Feedser.Exporter.Json/JsonExportSnapshotBuilder.cs @@ -0,0 +1,239 @@ +using System.Collections.Generic; +using System.Globalization; +using System.IO; +using System.Runtime.CompilerServices; +using System.Security.Cryptography; +using System.Text; +using System.Threading.Tasks; +using StellaOps.Feedser.Models; + +namespace StellaOps.Feedser.Exporter.Json; + +/// +/// Writes canonical advisory snapshots into a vuln-list style directory tree with deterministic ordering. +/// +public sealed class JsonExportSnapshotBuilder +{ + private static readonly Encoding Utf8NoBom = new UTF8Encoding(encoderShouldEmitUTF8Identifier: false); + private readonly JsonExportOptions _options; + private readonly IJsonExportPathResolver _pathResolver; + + public JsonExportSnapshotBuilder(JsonExportOptions options, IJsonExportPathResolver pathResolver) + { + _options = options ?? throw new ArgumentNullException(nameof(options)); + _pathResolver = pathResolver ?? throw new ArgumentNullException(nameof(pathResolver)); + } + + public Task WriteAsync( + IReadOnlyCollection advisories, + DateTimeOffset exportedAt, + string? exportName = null, + CancellationToken cancellationToken = default) + { + if (advisories is null) + { + throw new ArgumentNullException(nameof(advisories)); + } + + return WriteAsync(EnumerateAsync(advisories, cancellationToken), exportedAt, exportName, cancellationToken); + } + + public async Task WriteAsync( + IAsyncEnumerable advisories, + DateTimeOffset exportedAt, + string? exportName = null, + CancellationToken cancellationToken = default) + { + if (advisories is null) + { + throw new ArgumentNullException(nameof(advisories)); + } + + var exportDirectoryName = exportName ?? exportedAt.UtcDateTime.ToString(_options.DirectoryNameFormat, CultureInfo.InvariantCulture); + if (string.IsNullOrWhiteSpace(exportDirectoryName)) + { + throw new InvalidOperationException("Export directory name resolved to an empty string."); + } + + var exportRoot = EnsureDirectoryExists(Path.GetFullPath(_options.OutputRoot)); + TrySetDirectoryTimestamp(exportRoot, exportedAt); + var exportDirectory = Path.Combine(exportRoot, exportDirectoryName); + + if (Directory.Exists(exportDirectory)) + { + Directory.Delete(exportDirectory, recursive: true); + } + + Directory.CreateDirectory(exportDirectory); + TrySetDirectoryTimestamp(exportDirectory, exportedAt); + + var seen = new HashSet(StringComparer.OrdinalIgnoreCase); + var files = new List(); + long totalBytes = 0L; + var advisoryCount = 0; + + await foreach (var advisory in advisories.WithCancellation(cancellationToken)) + { + cancellationToken.ThrowIfCancellationRequested(); + + advisoryCount++; + var entry = Resolve(advisory); + if (!seen.Add(entry.RelativePath)) + { + throw new InvalidOperationException($"Multiple advisories resolved to the same path '{entry.RelativePath}'."); + } + + var destination = Combine(exportDirectory, entry.Segments); + var destinationDirectory = Path.GetDirectoryName(destination); + if (!string.IsNullOrEmpty(destinationDirectory)) + { + EnsureDirectoryExists(destinationDirectory); + TrySetDirectoryTimestamp(destinationDirectory, exportedAt); + } + var payload = SnapshotSerializer.ToSnapshot(entry.Advisory); + var bytes = Utf8NoBom.GetBytes(payload); + + await File.WriteAllBytesAsync(destination, bytes, cancellationToken).ConfigureAwait(false); + File.SetLastWriteTimeUtc(destination, exportedAt.UtcDateTime); + + var digest = ComputeDigest(bytes); + files.Add(new JsonExportFile(entry.RelativePath, bytes.LongLength, digest)); + totalBytes += bytes.LongLength; + } + + files.Sort(static (left, right) => string.CompareOrdinal(left.RelativePath, right.RelativePath)); + + return new JsonExportResult(exportDirectory, exportedAt, files, advisoryCount, totalBytes); + } + + private static async IAsyncEnumerable EnumerateAsync( + IEnumerable advisories, + [EnumeratorCancellation] CancellationToken cancellationToken) + { + foreach (var advisory in advisories) + { + cancellationToken.ThrowIfCancellationRequested(); + yield return advisory; + await Task.Yield(); + } + } + + private static string EnsureDirectoryExists(string directory) + { + if (string.IsNullOrWhiteSpace(directory)) + { + throw new ArgumentException("Directory path must be provided.", nameof(directory)); + } + + Directory.CreateDirectory(directory); + return directory; + } + + private static string Combine(string root, IReadOnlyList segments) + { + var parts = new string[segments.Count + 1]; + parts[0] = root; + for (var i = 0; i < segments.Count; i++) + { + parts[i + 1] = segments[i]; + } + + return Path.Combine(parts); + } + + private static void TrySetDirectoryTimestamp(string directory, DateTimeOffset timestamp) + { + try + { + Directory.SetLastWriteTimeUtc(directory, timestamp.UtcDateTime); + } + catch (IOException) + { + // Ignore failure to set timestamps; not critical for content determinism. + } + catch (UnauthorizedAccessException) + { + // Ignore permission issues when setting timestamps. + } + catch (PlatformNotSupportedException) + { + // Some platforms may not support this operation. + } + } + + private PathResolution Resolve(Advisory advisory) + { + if (advisory is null) + { + throw new ArgumentNullException(nameof(advisory)); + } + + var relativePath = _pathResolver.GetRelativePath(advisory); + var segments = NormalizeRelativePath(relativePath); + var normalized = string.Join('/', segments); + return new PathResolution(advisory, normalized, segments); + } + + private static string[] NormalizeRelativePath(string relativePath) + { + if (string.IsNullOrWhiteSpace(relativePath)) + { + throw new InvalidOperationException("Path resolver returned an empty path."); + } + + if (Path.IsPathRooted(relativePath)) + { + throw new InvalidOperationException("Path resolver returned an absolute path; only relative paths are supported."); + } + + var pieces = relativePath.Split(new[] { '/', '\\' }, StringSplitOptions.RemoveEmptyEntries); + if (pieces.Length == 0) + { + throw new InvalidOperationException("Path resolver produced no path segments."); + } + + var sanitized = new string[pieces.Length]; + for (var i = 0; i < pieces.Length; i++) + { + var segment = pieces[i]; + if (segment == "." || segment == "..") + { + throw new InvalidOperationException("Relative paths cannot include '.' or '..' segments."); + } + + sanitized[i] = SanitizeSegment(segment); + } + + return sanitized; + } + + private static string SanitizeSegment(string segment) + { + var invalid = Path.GetInvalidFileNameChars(); + Span buffer = stackalloc char[segment.Length]; + var count = 0; + foreach (var ch in segment) + { + if (ch == '/' || ch == '\\' || Array.IndexOf(invalid, ch) >= 0) + { + buffer[count++] = '_'; + } + else + { + buffer[count++] = ch; + } + } + + var sanitized = new string(buffer[..count]).Trim(); + return string.IsNullOrEmpty(sanitized) ? "_" : sanitized; + } + + private sealed record PathResolution(Advisory Advisory, string RelativePath, IReadOnlyList Segments); + + private static string ComputeDigest(ReadOnlySpan payload) + { + var hash = SHA256.HashData(payload); + var hex = Convert.ToHexString(hash).ToLowerInvariant(); + return $"sha256:{hex}"; + } +} diff --git a/src/StellaOps.Feedser.Exporter.Json/JsonExporterDependencyInjectionRoutine.cs b/src/StellaOps.Feedser.Exporter.Json/JsonExporterDependencyInjectionRoutine.cs new file mode 100644 index 00000000..67f5c655 --- /dev/null +++ b/src/StellaOps.Feedser.Exporter.Json/JsonExporterDependencyInjectionRoutine.cs @@ -0,0 +1,59 @@ +using System; +using System.IO; +using Microsoft.Extensions.Configuration; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.DependencyInjection.Extensions; +using Microsoft.Extensions.Options; +using StellaOps.DependencyInjection; +using StellaOps.Feedser.Core.Jobs; +using StellaOps.Feedser.Storage.Mongo.Exporting; + +namespace StellaOps.Feedser.Exporter.Json; + +public sealed class JsonExporterDependencyInjectionRoutine : IDependencyInjectionRoutine +{ + private const string ConfigurationSection = "feedser:exporters:json"; + + public IServiceCollection Register(IServiceCollection services, IConfiguration configuration) + { + ArgumentNullException.ThrowIfNull(services); + ArgumentNullException.ThrowIfNull(configuration); + + services.TryAddSingleton(); + services.TryAddSingleton(); + + services.AddOptions() + .Bind(configuration.GetSection(ConfigurationSection)) + .PostConfigure(static options => + { + if (string.IsNullOrWhiteSpace(options.OutputRoot)) + { + options.OutputRoot = Path.Combine("exports", "json"); + } + + if (string.IsNullOrWhiteSpace(options.DirectoryNameFormat)) + { + options.DirectoryNameFormat = "yyyyMMdd'T'HHmmss'Z'"; + } + }); + + services.AddSingleton(); + services.AddTransient(); + + services.PostConfigure(options => + { + if (!options.Definitions.ContainsKey(JsonExportJob.JobKind)) + { + options.Definitions[JsonExportJob.JobKind] = new JobDefinition( + JsonExportJob.JobKind, + typeof(JsonExportJob), + JsonExportJob.DefaultTimeout, + JsonExportJob.DefaultLeaseDuration, + null, + true); + } + }); + + return services; + } +} diff --git a/src/StellaOps.Feedser.Exporter.Json/JsonExporterPlugin.cs b/src/StellaOps.Feedser.Exporter.Json/JsonExporterPlugin.cs new file mode 100644 index 00000000..d03ab541 --- /dev/null +++ b/src/StellaOps.Feedser.Exporter.Json/JsonExporterPlugin.cs @@ -0,0 +1,23 @@ +using System; +using Microsoft.Extensions.DependencyInjection; +using StellaOps.Feedser.Storage.Mongo.Advisories; +using StellaOps.Plugin; + +namespace StellaOps.Feedser.Exporter.Json; + +public sealed class JsonExporterPlugin : IExporterPlugin +{ + public string Name => JsonFeedExporter.ExporterName; + + public bool IsAvailable(IServiceProvider services) + { + ArgumentNullException.ThrowIfNull(services); + return services.GetService() is not null; + } + + public IFeedExporter Create(IServiceProvider services) + { + ArgumentNullException.ThrowIfNull(services); + return ActivatorUtilities.CreateInstance(services); + } +} diff --git a/src/StellaOps.Feedser.Exporter.Json/JsonFeedExporter.cs b/src/StellaOps.Feedser.Exporter.Json/JsonFeedExporter.cs new file mode 100644 index 00000000..84c03d48 --- /dev/null +++ b/src/StellaOps.Feedser.Exporter.Json/JsonFeedExporter.cs @@ -0,0 +1,170 @@ +using System; +using System.Globalization; +using System.IO; +using System.Linq; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Options; +using StellaOps.Feedser.Storage.Mongo.Advisories; +using StellaOps.Feedser.Storage.Mongo.Exporting; +using StellaOps.Plugin; + +namespace StellaOps.Feedser.Exporter.Json; + +public sealed class JsonFeedExporter : IFeedExporter +{ + public const string ExporterName = "json"; + public const string ExporterId = "export:json"; + + private readonly IAdvisoryStore _advisoryStore; + private readonly JsonExportOptions _options; + private readonly IJsonExportPathResolver _pathResolver; + private readonly ExportStateManager _stateManager; + private readonly ILogger _logger; + private readonly TimeProvider _timeProvider; + private readonly string _exporterVersion; + + public JsonFeedExporter( + IAdvisoryStore advisoryStore, + IOptions options, + IJsonExportPathResolver pathResolver, + ExportStateManager stateManager, + ILogger logger, + TimeProvider? timeProvider = null) + { + _advisoryStore = advisoryStore ?? throw new ArgumentNullException(nameof(advisoryStore)); + _options = options?.Value ?? throw new ArgumentNullException(nameof(options)); + _pathResolver = pathResolver ?? throw new ArgumentNullException(nameof(pathResolver)); + _stateManager = stateManager ?? throw new ArgumentNullException(nameof(stateManager)); + _logger = logger ?? throw new ArgumentNullException(nameof(logger)); + _timeProvider = timeProvider ?? TimeProvider.System; + _exporterVersion = ExporterVersion.GetVersion(typeof(JsonFeedExporter)); + } + + public string Name => ExporterName; + + public async Task ExportAsync(IServiceProvider services, CancellationToken cancellationToken) + { + var exportedAt = _timeProvider.GetUtcNow(); + var exportId = exportedAt.ToString(_options.DirectoryNameFormat, CultureInfo.InvariantCulture); + var exportRoot = Path.GetFullPath(_options.OutputRoot); + + _logger.LogInformation("Starting JSON export {ExportId}", exportId); + + var existingState = await _stateManager.GetAsync(ExporterId, cancellationToken).ConfigureAwait(false); + + var builder = new JsonExportSnapshotBuilder(_options, _pathResolver); + var advisoryStream = _advisoryStore.StreamAsync(cancellationToken); + var result = await builder.WriteAsync(advisoryStream, exportedAt, exportId, cancellationToken).ConfigureAwait(false); + + var digest = ExportDigestCalculator.ComputeTreeDigest(result); + _logger.LogInformation( + "JSON export {ExportId} wrote {FileCount} files ({Bytes} bytes) covering {AdvisoryCount} advisories with digest {Digest}", + exportId, + result.Files.Length, + result.TotalBytes, + result.AdvisoryCount, + digest); + + var manifest = result.Files + .Select(static file => new ExportFileRecord(file.RelativePath, file.Length, file.Digest)) + .ToArray(); + + if (existingState is not null + && existingState.Files.Count > 0 + && string.Equals(existingState.LastFullDigest, digest, StringComparison.Ordinal)) + { + _logger.LogInformation("JSON export {ExportId} produced unchanged digest; skipping state update.", exportId); + TryDeleteDirectory(result.ExportDirectory); + return; + } + + var resetBaseline = existingState is null + || string.IsNullOrWhiteSpace(existingState.BaseExportId) + || string.IsNullOrWhiteSpace(existingState.BaseDigest); + + if (existingState is not null + && !string.IsNullOrWhiteSpace(_options.TargetRepository) + && !string.Equals(existingState.TargetRepository, _options.TargetRepository, StringComparison.Ordinal)) + { + resetBaseline = true; + } + + await _stateManager.StoreFullExportAsync( + ExporterId, + exportId, + digest, + cursor: digest, + targetRepository: _options.TargetRepository, + exporterVersion: _exporterVersion, + resetBaseline: resetBaseline, + manifest: manifest, + cancellationToken: cancellationToken).ConfigureAwait(false); + + await JsonExportManifestWriter.WriteAsync(result, digest, _exporterVersion, cancellationToken).ConfigureAwait(false); + + if (_options.MaintainLatestSymlink) + { + TryUpdateLatestSymlink(exportRoot, result.ExportDirectory); + } + } + + private void TryUpdateLatestSymlink(string exportRoot, string exportDirectory) + { + if (string.IsNullOrWhiteSpace(_options.LatestSymlinkName)) + { + return; + } + + var latestPath = Path.Combine(exportRoot, _options.LatestSymlinkName); + + try + { + if (Directory.Exists(latestPath) || File.Exists(latestPath)) + { + TryRemoveExistingPointer(latestPath); + } + + Directory.CreateSymbolicLink(latestPath, exportDirectory); + _logger.LogDebug("Updated latest JSON export pointer to {Target}", exportDirectory); + } + catch (Exception ex) when (ex is IOException or UnauthorizedAccessException or PlatformNotSupportedException) + { + _logger.LogWarning(ex, "Failed to update latest JSON export pointer at {LatestPath}", latestPath); + } + } + + private void TryRemoveExistingPointer(string latestPath) + { + try + { + var attributes = File.GetAttributes(latestPath); + if (attributes.HasFlag(FileAttributes.Directory)) + { + Directory.Delete(latestPath, recursive: false); + } + else + { + File.Delete(latestPath); + } + } + catch (Exception ex) when (ex is IOException or UnauthorizedAccessException) + { + _logger.LogWarning(ex, "Failed to remove existing latest pointer {LatestPath}", latestPath); + } + } + + private void TryDeleteDirectory(string path) + { + try + { + if (Directory.Exists(path)) + { + Directory.Delete(path, recursive: true); + } + } + catch (Exception ex) when (ex is IOException or UnauthorizedAccessException) + { + _logger.LogWarning(ex, "Failed to remove unchanged export directory {ExportDirectory}", path); + } + } +} diff --git a/src/StellaOps.Feedser.Exporter.Json/StellaOps.Feedser.Exporter.Json.csproj b/src/StellaOps.Feedser.Exporter.Json/StellaOps.Feedser.Exporter.Json.csproj new file mode 100644 index 00000000..72b8b2a8 --- /dev/null +++ b/src/StellaOps.Feedser.Exporter.Json/StellaOps.Feedser.Exporter.Json.csproj @@ -0,0 +1,22 @@ + + + + net10.0 + preview + enable + enable + true + + + + + + + + + + + + + + diff --git a/src/StellaOps.Feedser.Exporter.Json/TASKS.md b/src/StellaOps.Feedser.Exporter.Json/TASKS.md new file mode 100644 index 00000000..fbf69fe1 --- /dev/null +++ b/src/StellaOps.Feedser.Exporter.Json/TASKS.md @@ -0,0 +1,11 @@ +# TASKS +| Task | Owner(s) | Depends on | Notes | +|---|---|---|---| +|Directory layout strategy (vuln-list mirror)|BE-Export|Models|DONE – `VulnListJsonExportPathResolver` maps CVE, GHSA, distro, and vendor identifiers into vuln-list style paths.| +|Deterministic serializer|BE-Export|Models|DONE – Canonical serializer + snapshot builder emit stable JSON across runs.| +|ExportState read/write|BE-Export|Storage.Mongo|DONE – `JsonFeedExporter` reads prior state, stores digests/cursors, and skips unchanged exports.| +|JsonExportJob wiring|BE-Export|Core|DONE – Job scheduler options now configurable via DI; JSON job registered with scheduler.| +|Snapshot tests for file tree|QA|Exporters|DONE – Added resolver/exporter tests asserting tree layout and deterministic behavior.| +|Parity smoke vs upstream vuln-list|QA|Exporters|DONE – `JsonExporterParitySmokeTests` covers common ecosystems against vuln-list layout.| +|Stream advisories during export|BE-Export|Storage.Mongo|DONE – exporter + streaming-only test ensures single enumeration and per-file digest capture.| +|Emit export manifest with digest metadata|BE-Export|Exporters|DONE – manifest now includes per-file digests/sizes alongside tree digest.| diff --git a/src/StellaOps.Feedser.Exporter.Json/VulnListJsonExportPathResolver.cs b/src/StellaOps.Feedser.Exporter.Json/VulnListJsonExportPathResolver.cs new file mode 100644 index 00000000..7485e2ff --- /dev/null +++ b/src/StellaOps.Feedser.Exporter.Json/VulnListJsonExportPathResolver.cs @@ -0,0 +1,455 @@ +using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using System.Text.RegularExpressions; +using StellaOps.Feedser.Models; +using StellaOps.Feedser.Normalization.Identifiers; + +namespace StellaOps.Feedser.Exporter.Json; + +/// +/// Path resolver approximating the directory layout used by aquasecurity/vuln-list. +/// Handles common vendor, distro, and ecosystem shapes with deterministic fallbacks. +/// +public sealed class VulnListJsonExportPathResolver : IJsonExportPathResolver +{ + private static readonly Regex CvePattern = new("^CVE-(?\\d{4})-(?\\d{4,})$", RegexOptions.Compiled | RegexOptions.CultureInvariant | RegexOptions.IgnoreCase); + private static readonly Regex GhsaPattern = new("^GHSA-[0-9a-z]{4}-[0-9a-z]{4}-[0-9a-z]{4}$", RegexOptions.Compiled | RegexOptions.CultureInvariant | RegexOptions.IgnoreCase); + private static readonly Regex UsnPattern = new("^USN-(?\\d+-\\d+)(?[a-z])?$", RegexOptions.Compiled | RegexOptions.CultureInvariant | RegexOptions.IgnoreCase); + private static readonly Regex DebianPattern = new("^(?DLA|DSA|ELA)-(?\\d+-\\d+)$", RegexOptions.Compiled | RegexOptions.CultureInvariant | RegexOptions.IgnoreCase); + private static readonly Regex RedHatPattern = new("^RH(?SA|BA|EA)-(?[0-9:.-]+)$", RegexOptions.Compiled | RegexOptions.CultureInvariant | RegexOptions.IgnoreCase); + private static readonly Regex AmazonPattern = new("^ALAS(?2|2022|2023)?-(?[0-9A-Za-z:._-]+)$", RegexOptions.Compiled | RegexOptions.CultureInvariant | RegexOptions.IgnoreCase); + private static readonly Regex OraclePattern = new("^(?ELSA|ELBA|ELSA-OCI|ELBA-OCI)-(?[0-9A-Za-z:._-]+)$", RegexOptions.Compiled | RegexOptions.CultureInvariant | RegexOptions.IgnoreCase); + private static readonly Regex PhotonPattern = new("^PHSA-(?[0-9A-Za-z:._-]+)$", RegexOptions.Compiled | RegexOptions.CultureInvariant | RegexOptions.IgnoreCase); + private static readonly Regex RockyPattern = new("^RLSA-(?[0-9A-Za-z:._-]+)$", RegexOptions.Compiled | RegexOptions.CultureInvariant | RegexOptions.IgnoreCase); + private static readonly Regex SusePattern = new("^SUSE-(?SU|RU|OU|SB)-(?[0-9A-Za-z:._-]+)$", RegexOptions.Compiled | RegexOptions.CultureInvariant | RegexOptions.IgnoreCase); + + private static readonly Dictionary SourceDirectoryMap = new(StringComparer.OrdinalIgnoreCase) + { + ["nvd"] = new[] { "nvd" }, + ["ghsa"] = new[] { "ghsa" }, + ["github"] = new[] { "ghsa" }, + ["osv"] = new[] { "osv" }, + ["redhat"] = new[] { "redhat", "oval" }, + ["ubuntu"] = new[] { "ubuntu" }, + ["debian"] = new[] { "debian" }, + ["oracle"] = new[] { "oracle" }, + ["photon"] = new[] { "photon" }, + ["rocky"] = new[] { "rocky" }, + ["suse"] = new[] { "suse" }, + ["amazon"] = new[] { "amazon" }, + ["aws"] = new[] { "amazon" }, + ["alpine"] = new[] { "alpine" }, + ["wolfi"] = new[] { "wolfi" }, + ["chainguard"] = new[] { "chainguard" }, + ["cert-fr"] = new[] { "cert", "fr" }, + ["cert-in"] = new[] { "cert", "in" }, + ["cert-cc"] = new[] { "cert", "cc" }, + ["cert-bund"] = new[] { "cert", "bund" }, + ["cisa"] = new[] { "ics", "cisa" }, + ["ics-cisa"] = new[] { "ics", "cisa" }, + ["ics-kaspersky"] = new[] { "ics", "kaspersky" }, + ["kaspersky"] = new[] { "ics", "kaspersky" }, + }; + + private static readonly Dictionary GhsaEcosystemMap = new(StringComparer.OrdinalIgnoreCase) + { + ["go"] = "go", + ["golang"] = "go", + ["npm"] = "npm", + ["maven"] = "maven", + ["pypi"] = "pip", + ["pip"] = "pip", + ["nuget"] = "nuget", + ["composer"] = "composer", + ["packagist"] = "composer", + ["rubygems"] = "rubygems", + ["gem"] = "rubygems", + ["swift"] = "swift", + ["cargo"] = "cargo", + ["hex"] = "hex", + ["pub"] = "pub", + ["github"] = "github", + ["docker"] = "container", + }; + + public string GetRelativePath(Advisory advisory) + { + if (advisory is null) + { + throw new ArgumentNullException(nameof(advisory)); + } + + var identifier = SelectPreferredIdentifier(advisory); + if (identifier.Length == 0) + { + throw new InvalidOperationException("Unable to derive identifier for advisory."); + } + + var layout = ResolveLayout(advisory, identifier); + var segments = new string[layout.Segments.Length + 1]; + for (var i = 0; i < layout.Segments.Length; i++) + { + segments[i] = layout.Segments[i]; + } + segments[^1] = layout.FileName; + return Path.Combine(segments); + } + + private static Layout ResolveLayout(Advisory advisory, string identifier) + { + if (TryResolveCve(identifier, out var layout)) + { + return layout; + } + + if (TryResolveGhsa(advisory, identifier, out layout)) + { + return layout; + } + + if (TryResolveUsn(identifier, out layout) || + TryResolveDebian(identifier, out layout) || + TryResolveRedHat(identifier, out layout) || + TryResolveAmazon(identifier, out layout) || + TryResolveOracle(identifier, out layout) || + TryResolvePhoton(identifier, out layout) || + TryResolveRocky(identifier, out layout) || + TryResolveSuse(identifier, out layout)) + { + return layout; + } + + if (TryResolveByProvenance(advisory, identifier, out layout)) + { + return layout; + } + + return new Layout(new[] { "misc" }, CreateFileName(identifier)); + } + + private static bool TryResolveCve(string identifier, out Layout layout) + { + var match = CvePattern.Match(identifier); + if (!match.Success) + { + layout = default; + return false; + } + + var year = match.Groups["year"].Value; + layout = new Layout(new[] { "nvd", year }, CreateFileName(identifier, uppercase: true)); + return true; + } + + private static bool TryResolveGhsa(Advisory advisory, string identifier, out Layout layout) + { + if (!GhsaPattern.IsMatch(identifier)) + { + layout = default; + return false; + } + + if (TryGetGhsaPackage(advisory, out var ecosystem, out var packagePath)) + { + layout = new Layout(new[] { "ghsa", ecosystem, packagePath }, CreateFileName(identifier, uppercase: true)); + return true; + } + + layout = new Layout(new[] { "github", "advisories" }, CreateFileName(identifier, uppercase: true)); + return true; + } + + private static bool TryResolveUsn(string identifier, out Layout layout) + { + if (!UsnPattern.IsMatch(identifier)) + { + layout = default; + return false; + } + + layout = new Layout(new[] { "ubuntu" }, CreateFileName(identifier, uppercase: true)); + return true; + } + + private static bool TryResolveDebian(string identifier, out Layout layout) + { + var match = DebianPattern.Match(identifier); + if (!match.Success) + { + layout = default; + return false; + } + + layout = new Layout(new[] { "debian" }, CreateFileName(identifier, uppercase: true)); + return true; + } + + private static bool TryResolveRedHat(string identifier, out Layout layout) + { + if (!RedHatPattern.IsMatch(identifier)) + { + layout = default; + return false; + } + + layout = new Layout(new[] { "redhat", "oval" }, CreateFileName(identifier, uppercase: true)); + return true; + } + + private static bool TryResolveAmazon(string identifier, out Layout layout) + { + var match = AmazonPattern.Match(identifier); + if (!match.Success) + { + layout = default; + return false; + } + + var channel = match.Groups["channel"].Value; + var subdirectory = channel switch + { + "2" => "2", + "2023" => "2023", + "2022" => "2022", + _ => "1", + }; + + layout = new Layout(new[] { "amazon", subdirectory }, CreateFileName(identifier, uppercase: true)); + return true; + } + + private static bool TryResolveOracle(string identifier, out Layout layout) + { + if (!OraclePattern.IsMatch(identifier)) + { + layout = default; + return false; + } + + layout = new Layout(new[] { "oracle", "linux" }, CreateFileName(identifier, uppercase: true)); + return true; + } + + private static bool TryResolvePhoton(string identifier, out Layout layout) + { + if (!PhotonPattern.IsMatch(identifier)) + { + layout = default; + return false; + } + + layout = new Layout(new[] { "photon" }, CreateFileName(identifier, uppercase: true)); + return true; + } + + private static bool TryResolveRocky(string identifier, out Layout layout) + { + if (!RockyPattern.IsMatch(identifier)) + { + layout = default; + return false; + } + + layout = new Layout(new[] { "rocky" }, CreateFileName(identifier, uppercase: true)); + return true; + } + + private static bool TryResolveSuse(string identifier, out Layout layout) + { + if (!SusePattern.IsMatch(identifier)) + { + layout = default; + return false; + } + + layout = new Layout(new[] { "suse" }, CreateFileName(identifier, uppercase: true)); + return true; + } + + private static bool TryResolveByProvenance(Advisory advisory, string identifier, out Layout layout) + { + foreach (var source in EnumerateDistinctProvenanceSources(advisory)) + { + if (SourceDirectoryMap.TryGetValue(source, out var segments)) + { + layout = new Layout(segments, CreateFileName(identifier)); + return true; + } + } + + layout = default; + return false; + } + + private static bool TryGetGhsaPackage(Advisory advisory, out string ecosystem, out string packagePath) + { + foreach (var package in advisory.AffectedPackages) + { + if (!TryParsePackageUrl(package.Identifier, out var type, out var encodedPath)) + { + continue; + } + + if (GhsaEcosystemMap.TryGetValue(type, out var mapped)) + { + ecosystem = mapped; + } + else + { + ecosystem = type.ToLowerInvariant(); + } + + packagePath = encodedPath; + return true; + } + + ecosystem = "advisories"; + packagePath = "_"; + return false; + } + + private static bool TryParsePackageUrl(string identifier, out string type, out string encodedPath) + { + type = string.Empty; + encodedPath = string.Empty; + + if (!IdentifierNormalizer.TryNormalizePackageUrl(identifier, out _, out var packageUrl)) + { + return false; + } + + var segments = packageUrl!.NamespaceSegments.IsDefaultOrEmpty + ? new[] { packageUrl.Name } + : packageUrl.NamespaceSegments.Append(packageUrl.Name).ToArray(); + + type = packageUrl.Type; + encodedPath = string.Join("%2F", segments); + return true; + } + + private static string CreateFileName(string identifier, bool uppercase = false) + { + var candidate = uppercase ? identifier.ToUpperInvariant() : identifier; + return $"{SanitizeFileName(candidate)}.json"; + } + + private static IEnumerable EnumerateDistinctProvenanceSources(Advisory advisory) + { + var seen = new HashSet(StringComparer.OrdinalIgnoreCase); + + foreach (var source in advisory.Provenance) + { + if (TryAddSource(source.Source)) + { + yield return source.Source; + } + } + + foreach (var reference in advisory.References) + { + if (TryAddSource(reference.Provenance.Source)) + { + yield return reference.Provenance.Source; + } + } + + foreach (var package in advisory.AffectedPackages) + { + foreach (var source in package.Provenance) + { + if (TryAddSource(source.Source)) + { + yield return source.Source; + } + } + + foreach (var range in package.VersionRanges) + { + if (TryAddSource(range.Provenance.Source)) + { + yield return range.Provenance.Source; + } + } + } + + foreach (var metric in advisory.CvssMetrics) + { + if (TryAddSource(metric.Provenance.Source)) + { + yield return metric.Provenance.Source; + } + } + + bool TryAddSource(string? value) + { + if (string.IsNullOrWhiteSpace(value)) + { + return false; + } + + return seen.Add(value); + } + } + + private static string SelectPreferredIdentifier(Advisory advisory) + { + if (TrySelectIdentifier(advisory.AdvisoryKey, out var preferred)) + { + return preferred; + } + + foreach (var alias in advisory.Aliases) + { + if (TrySelectIdentifier(alias, out preferred)) + { + return preferred; + } + } + + return advisory.AdvisoryKey.Trim(); + } + + private static bool TrySelectIdentifier(string value, out string identifier) + { + identifier = string.Empty; + if (string.IsNullOrWhiteSpace(value)) + { + return false; + } + + var trimmed = value.Trim(); + if (CvePattern.IsMatch(trimmed) || GhsaPattern.IsMatch(trimmed)) + { + identifier = trimmed; + return true; + } + + identifier = trimmed; + return false; + } + + private static string SanitizeFileName(string name) + { + var invalid = Path.GetInvalidFileNameChars(); + Span buffer = stackalloc char[name.Length]; + var count = 0; + foreach (var ch in name) + { + if (ch == '/' || ch == '\\' || Array.IndexOf(invalid, ch) >= 0) + { + buffer[count++] = '_'; + } + else + { + buffer[count++] = ch; + } + } + + var sanitized = new string(buffer[..count]).Trim(); + return string.IsNullOrEmpty(sanitized) ? "advisory" : sanitized; + } + + private readonly record struct Layout(string[] Segments, string FileName); +} diff --git a/src/StellaOps.Feedser.Exporter.TrivyDb.Tests/StellaOps.Feedser.Exporter.TrivyDb.Tests.csproj b/src/StellaOps.Feedser.Exporter.TrivyDb.Tests/StellaOps.Feedser.Exporter.TrivyDb.Tests.csproj new file mode 100644 index 00000000..6ac1cb21 --- /dev/null +++ b/src/StellaOps.Feedser.Exporter.TrivyDb.Tests/StellaOps.Feedser.Exporter.TrivyDb.Tests.csproj @@ -0,0 +1,13 @@ + + + net10.0 + enable + enable + + + + + + + + diff --git a/src/StellaOps.Feedser.Exporter.TrivyDb.Tests/TrivyDbExportPlannerTests.cs b/src/StellaOps.Feedser.Exporter.TrivyDb.Tests/TrivyDbExportPlannerTests.cs new file mode 100644 index 00000000..5b58d2ea --- /dev/null +++ b/src/StellaOps.Feedser.Exporter.TrivyDb.Tests/TrivyDbExportPlannerTests.cs @@ -0,0 +1,86 @@ +using System; +using StellaOps.Feedser.Exporter.TrivyDb; +using StellaOps.Feedser.Storage.Mongo.Exporting; + +namespace StellaOps.Feedser.Exporter.TrivyDb.Tests; + +public sealed class TrivyDbExportPlannerTests +{ + [Fact] + public void CreatePlan_ReturnsFullWhenStateMissing() + { + var planner = new TrivyDbExportPlanner(); + var manifest = new[] { new ExportFileRecord("path.json", 10, "sha256:a") }; + var plan = planner.CreatePlan(existingState: null, treeDigest: "sha256:abcd", manifest); + + Assert.Equal(TrivyDbExportMode.Full, plan.Mode); + Assert.Equal("sha256:abcd", plan.TreeDigest); + Assert.Null(plan.BaseExportId); + Assert.Null(plan.BaseManifestDigest); + Assert.True(plan.ResetBaseline); + Assert.Equal(manifest, plan.Manifest); + } + + [Fact] + public void CreatePlan_ReturnsSkipWhenCursorMatches() + { + var planner = new TrivyDbExportPlanner(); + var existingManifest = new[] { new ExportFileRecord("path.json", 10, "sha256:a") }; + var state = new ExportStateRecord( + Id: TrivyDbFeedExporter.ExporterId, + BaseExportId: "20240810T000000Z", + BaseDigest: "sha256:base", + LastFullDigest: "sha256:base", + LastDeltaDigest: null, + ExportCursor: "sha256:unchanged", + TargetRepository: "feedser/trivy", + ExporterVersion: "1.0", + UpdatedAt: DateTimeOffset.UtcNow, + Files: existingManifest); + + var plan = planner.CreatePlan(state, "sha256:unchanged", existingManifest); + + Assert.Equal(TrivyDbExportMode.Skip, plan.Mode); + Assert.Equal("sha256:unchanged", plan.TreeDigest); + Assert.Equal("20240810T000000Z", plan.BaseExportId); + Assert.Equal("sha256:base", plan.BaseManifestDigest); + Assert.False(plan.ResetBaseline); + Assert.Empty(plan.ChangedFiles); + Assert.Empty(plan.RemovedPaths); + } + + [Fact] + public void CreatePlan_ReturnsFullWhenCursorDiffers() + { + var planner = new TrivyDbExportPlanner(); + var manifest = new[] { new ExportFileRecord("path.json", 10, "sha256:a") }; + var state = new ExportStateRecord( + Id: TrivyDbFeedExporter.ExporterId, + BaseExportId: "20240810T000000Z", + BaseDigest: "sha256:base", + LastFullDigest: "sha256:base", + LastDeltaDigest: null, + ExportCursor: "sha256:old", + TargetRepository: "feedser/trivy", + ExporterVersion: "1.0", + UpdatedAt: DateTimeOffset.UtcNow, + Files: manifest); + + var newManifest = new[] { new ExportFileRecord("path.json", 10, "sha256:b") }; + var plan = planner.CreatePlan(state, "sha256:new", newManifest); + + Assert.Equal(TrivyDbExportMode.Delta, plan.Mode); + Assert.Equal("sha256:new", plan.TreeDigest); + Assert.Equal("20240810T000000Z", plan.BaseExportId); + Assert.Equal("sha256:base", plan.BaseManifestDigest); + Assert.False(plan.ResetBaseline); + Assert.Single(plan.ChangedFiles); + + var deltaState = state with { LastDeltaDigest = "sha256:delta" }; + var deltaPlan = planner.CreatePlan(deltaState, "sha256:newer", newManifest); + + Assert.Equal(TrivyDbExportMode.Full, deltaPlan.Mode); + Assert.True(deltaPlan.ResetBaseline); + Assert.Equal(deltaPlan.Manifest, deltaPlan.ChangedFiles); + } +} diff --git a/src/StellaOps.Feedser.Exporter.TrivyDb.Tests/TrivyDbFeedExporterTests.cs b/src/StellaOps.Feedser.Exporter.TrivyDb.Tests/TrivyDbFeedExporterTests.cs new file mode 100644 index 00000000..c9ae1176 --- /dev/null +++ b/src/StellaOps.Feedser.Exporter.TrivyDb.Tests/TrivyDbFeedExporterTests.cs @@ -0,0 +1,1027 @@ +using System; +using System.Collections.Generic; +using System.Globalization; +using System.Linq; +using System.Runtime.CompilerServices; +using System.Security.Cryptography; +using System.Text; +using System.Text.Json; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Logging.Abstractions; +using Microsoft.Extensions.Options; +using StellaOps.Feedser.Exporter.Json; +using StellaOps.Feedser.Exporter.TrivyDb; +using StellaOps.Feedser.Models; +using StellaOps.Feedser.Storage.Mongo.Advisories; +using StellaOps.Feedser.Storage.Mongo.Exporting; + +namespace StellaOps.Feedser.Exporter.TrivyDb.Tests; + +public sealed class TrivyDbFeedExporterTests : IDisposable +{ + private readonly string _root; + private readonly string _jsonRoot; + + public TrivyDbFeedExporterTests() + { + _root = Directory.CreateTempSubdirectory("feedser-trivy-exporter-tests").FullName; + _jsonRoot = Path.Combine(_root, "tree"); + } + + [Fact] + public async Task ExportAsync_SortsAdvisoriesByKeyDeterministically() + { + var advisoryB = CreateSampleAdvisory("CVE-2024-1002", "Second advisory"); + var advisoryA = CreateSampleAdvisory("CVE-2024-1001", "First advisory"); + + var advisoryStore = new StubAdvisoryStore(advisoryB, advisoryA); + + var optionsValue = new TrivyDbExportOptions + { + OutputRoot = _root, + ReferencePrefix = "example/trivy", + KeepWorkingTree = false, + Json = new JsonExportOptions + { + OutputRoot = _jsonRoot, + MaintainLatestSymlink = false, + }, + }; + + var options = Options.Create(optionsValue); + var packageBuilder = new TrivyDbPackageBuilder(); + var ociWriter = new TrivyDbOciWriter(); + var planner = new TrivyDbExportPlanner(); + var stateStore = new InMemoryExportStateStore(); + var timeProvider = new TestTimeProvider(DateTimeOffset.Parse("2024-09-20T00:00:00Z", CultureInfo.InvariantCulture)); + var stateManager = new ExportStateManager(stateStore, timeProvider); + var builderMetadata = JsonSerializer.SerializeToUtf8Bytes(new + { + Version = 2, + NextUpdate = "2024-09-21T00:00:00Z", + UpdatedAt = "2024-09-20T00:00:00Z", + }); + + var recordingBuilder = new RecordingTrivyDbBuilder(_root, builderMetadata); + var orasPusher = new StubTrivyDbOrasPusher(); + var exporter = new TrivyDbFeedExporter( + advisoryStore, + new VulnListJsonExportPathResolver(), + options, + packageBuilder, + ociWriter, + stateManager, + planner, + recordingBuilder, + orasPusher, + NullLogger.Instance, + timeProvider); + + using var provider = new ServiceCollection().BuildServiceProvider(); + await exporter.ExportAsync(provider, CancellationToken.None); + + var paths = recordingBuilder.LastRelativePaths; + Assert.NotNull(paths); + + var sorted = paths!.OrderBy(static p => p, StringComparer.Ordinal).ToArray(); + Assert.Equal(sorted, paths); + + advisoryStore.SetAdvisories(advisoryA, advisoryB); + timeProvider.Advance(TimeSpan.FromMinutes(7)); + await exporter.ExportAsync(provider, CancellationToken.None); + + var record = await stateStore.FindAsync(TrivyDbFeedExporter.ExporterId, CancellationToken.None); + Assert.NotNull(record); + Assert.Equal("20240920T000000Z", record!.BaseExportId); + Assert.Single(recordingBuilder.ManifestDigests); + } + + [Fact] + public async Task ExportAsync_SmallDatasetProducesDeterministicOciLayout() + { + var advisories = new[] + { + CreateSampleAdvisory("CVE-2024-3000", "Demo advisory 1"), + CreateSampleAdvisory("CVE-2024-3001", "Demo advisory 2"), + }; + + var run1 = await RunDeterministicExportAsync(advisories); + var run2 = await RunDeterministicExportAsync(advisories); + + Assert.Equal(run1.ManifestDigest, run2.ManifestDigest); + Assert.Equal(run1.IndexJson, run2.IndexJson); + Assert.Equal(run1.MetadataJson, run2.MetadataJson); + Assert.Equal(run1.ManifestJson, run2.ManifestJson); + + var digests1 = run1.Blobs.Keys.OrderBy(static d => d, StringComparer.Ordinal).ToArray(); + var digests2 = run2.Blobs.Keys.OrderBy(static d => d, StringComparer.Ordinal).ToArray(); + Assert.Equal(digests1, digests2); + + foreach (var digest in digests1) + { + Assert.True(run2.Blobs.TryGetValue(digest, out var other), $"Missing digest {digest} in second run"); + Assert.True(run1.Blobs[digest].SequenceEqual(other), $"Blob {digest} differs between runs"); + } + + using var metadataDoc = JsonDocument.Parse(run1.MetadataJson); + Assert.Equal(2, metadataDoc.RootElement.GetProperty("advisoryCount").GetInt32()); + + using var manifestDoc = JsonDocument.Parse(run1.ManifestJson); + Assert.Equal(TrivyDbMediaTypes.TrivyConfig, manifestDoc.RootElement.GetProperty("config").GetProperty("mediaType").GetString()); + var layer = manifestDoc.RootElement.GetProperty("layers")[0]; + Assert.Equal(TrivyDbMediaTypes.TrivyLayer, layer.GetProperty("mediaType").GetString()); + } + + [Fact] + public void ExportOptions_GetExportRoot_NormalizesRelativeRoot() + { + var options = new TrivyDbExportOptions + { + OutputRoot = Path.Combine("..", "exports", "trivy-test"), + }; + + var exportId = "20240901T000000Z"; + var path = options.GetExportRoot(exportId); + + Assert.True(Path.IsPathRooted(path)); + Assert.EndsWith(Path.Combine("exports", "trivy-test", exportId), path, StringComparison.Ordinal); + } + + [Fact] + public async Task ExportAsync_PersistsStateAndSkipsWhenDigestUnchanged() + { + var advisory = CreateSampleAdvisory(); + var advisoryStore = new StubAdvisoryStore(advisory); + + var optionsValue = new TrivyDbExportOptions + { + OutputRoot = _root, + ReferencePrefix = "example/trivy", + Json = new JsonExportOptions + { + OutputRoot = _jsonRoot, + MaintainLatestSymlink = false, + }, + KeepWorkingTree = false, + }; + + var options = Options.Create(optionsValue); + var packageBuilder = new TrivyDbPackageBuilder(); + var ociWriter = new TrivyDbOciWriter(); + var planner = new TrivyDbExportPlanner(); + var stateStore = new InMemoryExportStateStore(); + var timeProvider = new TestTimeProvider(DateTimeOffset.Parse("2024-09-01T00:00:00Z", CultureInfo.InvariantCulture)); + var stateManager = new ExportStateManager(stateStore, timeProvider); + var builderMetadata = JsonSerializer.SerializeToUtf8Bytes(new + { + Version = 2, + NextUpdate = "2024-09-02T00:00:00Z", + UpdatedAt = "2024-09-01T00:00:00Z", + }); + var builder = new StubTrivyDbBuilder(_root, builderMetadata); + var orasPusher = new StubTrivyDbOrasPusher(); + var exporter = new TrivyDbFeedExporter( + advisoryStore, + new VulnListJsonExportPathResolver(), + options, + packageBuilder, + ociWriter, + stateManager, + planner, + builder, + orasPusher, + NullLogger.Instance, + timeProvider); + + using var provider = new ServiceCollection().BuildServiceProvider(); + await exporter.ExportAsync(provider, CancellationToken.None); + + var record = await stateStore.FindAsync(TrivyDbFeedExporter.ExporterId, CancellationToken.None); + Assert.NotNull(record); + Assert.Equal("20240901T000000Z", record!.BaseExportId); + Assert.False(string.IsNullOrEmpty(record.ExportCursor)); + + var baseExportId = record.BaseExportId ?? string.Empty; + Assert.False(string.IsNullOrEmpty(baseExportId)); + var firstExportDirectory = Path.Combine(_root, baseExportId); + Assert.True(Directory.Exists(firstExportDirectory)); + + timeProvider.Advance(TimeSpan.FromMinutes(5)); + await exporter.ExportAsync(provider, CancellationToken.None); + + var updatedRecord = await stateStore.FindAsync(TrivyDbFeedExporter.ExporterId, CancellationToken.None); + Assert.NotNull(updatedRecord); + Assert.Equal(record.UpdatedAt, updatedRecord!.UpdatedAt); + Assert.Equal(record.LastFullDigest, updatedRecord.LastFullDigest); + + var skippedExportDirectory = Path.Combine(_root, "20240901T000500Z"); + Assert.False(Directory.Exists(skippedExportDirectory)); + + Assert.Empty(orasPusher.Pushes); + } + + [Fact] + public async Task ExportAsync_CreatesOfflineBundle() + { + var advisory = CreateSampleAdvisory(); + var advisoryStore = new StubAdvisoryStore(advisory); + + var optionsValue = new TrivyDbExportOptions + { + OutputRoot = _root, + ReferencePrefix = "example/trivy", + Json = new JsonExportOptions + { + OutputRoot = _jsonRoot, + MaintainLatestSymlink = false, + }, + KeepWorkingTree = false, + OfflineBundle = new TrivyDbOfflineBundleOptions + { + Enabled = true, + FileName = "{exportId}.bundle.tar.gz", + }, + }; + + var options = Options.Create(optionsValue); + var packageBuilder = new TrivyDbPackageBuilder(); + var ociWriter = new TrivyDbOciWriter(); + var planner = new TrivyDbExportPlanner(); + var stateStore = new InMemoryExportStateStore(); + var timeProvider = new TestTimeProvider(DateTimeOffset.Parse("2024-09-15T00:00:00Z", CultureInfo.InvariantCulture)); + var stateManager = new ExportStateManager(stateStore, timeProvider); + var builderMetadata = JsonSerializer.SerializeToUtf8Bytes(new + { + Version = 2, + NextUpdate = "2024-09-16T00:00:00Z", + UpdatedAt = "2024-09-15T00:00:00Z", + }); + var builder = new StubTrivyDbBuilder(_root, builderMetadata); + var orasPusher = new StubTrivyDbOrasPusher(); + var exporter = new TrivyDbFeedExporter( + advisoryStore, + new VulnListJsonExportPathResolver(), + options, + packageBuilder, + ociWriter, + stateManager, + planner, + builder, + orasPusher, + NullLogger.Instance, + timeProvider); + + using var provider = new ServiceCollection().BuildServiceProvider(); + await exporter.ExportAsync(provider, CancellationToken.None); + + var exportId = "20240915T000000Z"; + var bundlePath = Path.Combine(_root, $"{exportId}.bundle.tar.gz"); + Assert.True(File.Exists(bundlePath)); + Assert.Empty(orasPusher.Pushes); + } + + [Fact] + public async Task ExportAsync_SkipsOrasPushWhenDeltaPublishingDisabled() + { + var initial = CreateSampleAdvisory("CVE-2024-7100", "Publish toggles"); + var updated = CreateSampleAdvisory("CVE-2024-7100", "Publish toggles delta"); + var advisoryStore = new StubAdvisoryStore(initial); + + var optionsValue = new TrivyDbExportOptions + { + OutputRoot = _root, + ReferencePrefix = "example/trivy", + Json = new JsonExportOptions + { + OutputRoot = _jsonRoot, + MaintainLatestSymlink = false, + }, + KeepWorkingTree = true, + }; + + optionsValue.Oras.Enabled = true; + optionsValue.Oras.PublishFull = false; + optionsValue.Oras.PublishDelta = false; + + var options = Options.Create(optionsValue); + var packageBuilder = new TrivyDbPackageBuilder(); + var ociWriter = new TrivyDbOciWriter(); + var planner = new TrivyDbExportPlanner(); + var stateStore = new InMemoryExportStateStore(); + var timeProvider = new TestTimeProvider(DateTimeOffset.Parse("2024-10-20T00:00:00Z", CultureInfo.InvariantCulture)); + var stateManager = new ExportStateManager(stateStore, timeProvider); + var builderMetadata = JsonSerializer.SerializeToUtf8Bytes(new + { + Version = 2, + NextUpdate = "2024-10-21T00:00:00Z", + UpdatedAt = "2024-10-20T00:00:00Z", + }); + var builder = new StubTrivyDbBuilder(_root, builderMetadata); + var orasPusher = new StubTrivyDbOrasPusher(); + var exporter = new TrivyDbFeedExporter( + advisoryStore, + new VulnListJsonExportPathResolver(), + options, + packageBuilder, + ociWriter, + stateManager, + planner, + builder, + orasPusher, + NullLogger.Instance, + timeProvider); + + using var provider = new ServiceCollection().BuildServiceProvider(); + await exporter.ExportAsync(provider, CancellationToken.None); + + advisoryStore.SetAdvisories(updated); + timeProvider.Advance(TimeSpan.FromMinutes(15)); + await exporter.ExportAsync(provider, CancellationToken.None); + + Assert.Empty(orasPusher.Pushes); + } + + [Fact] + public async Task ExportAsync_SkipsOfflineBundleForDeltaWhenDisabled() + { + var initial = CreateSampleAdvisory("CVE-2024-7200", "Offline delta toggles"); + var updated = CreateSampleAdvisory("CVE-2024-7200", "Offline delta toggles updated"); + var advisoryStore = new StubAdvisoryStore(initial); + + var optionsValue = new TrivyDbExportOptions + { + OutputRoot = _root, + ReferencePrefix = "example/trivy", + Json = new JsonExportOptions + { + OutputRoot = _jsonRoot, + MaintainLatestSymlink = false, + }, + KeepWorkingTree = true, + OfflineBundle = new TrivyDbOfflineBundleOptions + { + Enabled = true, + IncludeFull = true, + IncludeDelta = false, + FileName = "{exportId}.bundle.tar.gz", + }, + }; + + var options = Options.Create(optionsValue); + var packageBuilder = new TrivyDbPackageBuilder(); + var ociWriter = new TrivyDbOciWriter(); + var planner = new TrivyDbExportPlanner(); + var stateStore = new InMemoryExportStateStore(); + var timeProvider = new TestTimeProvider(DateTimeOffset.Parse("2024-10-21T00:00:00Z", CultureInfo.InvariantCulture)); + var stateManager = new ExportStateManager(stateStore, timeProvider); + var builderMetadata = JsonSerializer.SerializeToUtf8Bytes(new + { + Version = 2, + NextUpdate = "2024-10-22T00:00:00Z", + UpdatedAt = "2024-10-21T00:00:00Z", + }); + var builder = new StubTrivyDbBuilder(_root, builderMetadata); + var orasPusher = new StubTrivyDbOrasPusher(); + var exporter = new TrivyDbFeedExporter( + advisoryStore, + new VulnListJsonExportPathResolver(), + options, + packageBuilder, + ociWriter, + stateManager, + planner, + builder, + orasPusher, + NullLogger.Instance, + timeProvider); + + using var provider = new ServiceCollection().BuildServiceProvider(); + await exporter.ExportAsync(provider, CancellationToken.None); + + var fullExportId = timeProvider.GetUtcNow().ToString(optionsValue.TagFormat, CultureInfo.InvariantCulture); + var fullBundlePath = Path.Combine(_root, $"{fullExportId}.bundle.tar.gz"); + Assert.True(File.Exists(fullBundlePath)); + + advisoryStore.SetAdvisories(updated); + timeProvider.Advance(TimeSpan.FromMinutes(10)); + await exporter.ExportAsync(provider, CancellationToken.None); + + var deltaExportId = timeProvider.GetUtcNow().ToString(optionsValue.TagFormat, CultureInfo.InvariantCulture); + var deltaBundlePath = Path.Combine(_root, $"{deltaExportId}.bundle.tar.gz"); + Assert.False(File.Exists(deltaBundlePath)); + } + + [Fact] + public async Task ExportAsync_ResetsBaselineWhenDeltaChainExists() + { + var advisory = CreateSampleAdvisory("CVE-2024-5000", "Baseline reset"); + var advisoryStore = new StubAdvisoryStore(advisory); + + var optionsValue = new TrivyDbExportOptions + { + OutputRoot = _root, + ReferencePrefix = "example/trivy", + Json = new JsonExportOptions + { + OutputRoot = _jsonRoot, + MaintainLatestSymlink = false, + }, + KeepWorkingTree = false, + TargetRepository = "registry.example/trivy", + }; + + var options = Options.Create(optionsValue); + var packageBuilder = new TrivyDbPackageBuilder(); + var ociWriter = new TrivyDbOciWriter(); + var planner = new TrivyDbExportPlanner(); + var stateStore = new InMemoryExportStateStore(); + var timeProvider = new TestTimeProvider(DateTimeOffset.Parse("2024-09-22T00:00:00Z", CultureInfo.InvariantCulture)); + var existingRecord = new ExportStateRecord( + TrivyDbFeedExporter.ExporterId, + BaseExportId: "20240919T120000Z", + BaseDigest: "sha256:base", + LastFullDigest: "sha256:base", + LastDeltaDigest: "sha256:delta", + ExportCursor: "sha256:old", + TargetRepository: "registry.example/trivy", + ExporterVersion: "0.9.0", + UpdatedAt: timeProvider.GetUtcNow().AddMinutes(-30), + Files: Array.Empty()); + await stateStore.UpsertAsync(existingRecord, CancellationToken.None); + + var stateManager = new ExportStateManager(stateStore, timeProvider); + var builderMetadata = JsonSerializer.SerializeToUtf8Bytes(new + { + Version = 2, + NextUpdate = "2024-09-23T00:00:00Z", + UpdatedAt = "2024-09-22T00:00:00Z", + }); + var builder = new StubTrivyDbBuilder(_root, builderMetadata); + var orasPusher = new StubTrivyDbOrasPusher(); + var exporter = new TrivyDbFeedExporter( + advisoryStore, + new VulnListJsonExportPathResolver(), + options, + packageBuilder, + ociWriter, + stateManager, + planner, + builder, + orasPusher, + NullLogger.Instance, + timeProvider); + + using var provider = new ServiceCollection().BuildServiceProvider(); + await exporter.ExportAsync(provider, CancellationToken.None); + + var updated = await stateStore.FindAsync(TrivyDbFeedExporter.ExporterId, CancellationToken.None); + Assert.NotNull(updated); + Assert.Equal("20240922T000000Z", updated!.BaseExportId); + Assert.Equal(updated.BaseDigest, updated.LastFullDigest); + Assert.Null(updated.LastDeltaDigest); + Assert.NotEqual("sha256:old", updated.ExportCursor); + Assert.Equal("registry.example/trivy", updated.TargetRepository); + Assert.NotEmpty(updated.Files); + } + + [Fact] + public async Task ExportAsync_DeltaSequencePromotesBaselineReset() + { + var baseline = CreateSampleAdvisory("CVE-2024-8100", "Baseline advisory"); + var firstDelta = CreateSampleAdvisory("CVE-2024-8100", "Baseline advisory updated"); + var secondDelta = CreateSampleAdvisory("CVE-2024-8200", "New advisory triggers full rebuild"); + + var advisoryStore = new StubAdvisoryStore(baseline); + + var optionsValue = new TrivyDbExportOptions + { + OutputRoot = _root, + ReferencePrefix = "example/trivy", + KeepWorkingTree = true, + Json = new JsonExportOptions + { + OutputRoot = _jsonRoot, + MaintainLatestSymlink = false, + }, + }; + + var options = Options.Create(optionsValue); + var packageBuilder = new TrivyDbPackageBuilder(); + var ociWriter = new TrivyDbOciWriter(); + var planner = new TrivyDbExportPlanner(); + var stateStore = new InMemoryExportStateStore(); + var timeProvider = new TestTimeProvider(DateTimeOffset.Parse("2024-11-01T00:00:00Z", CultureInfo.InvariantCulture)); + var stateManager = new ExportStateManager(stateStore, timeProvider); + var builderMetadata = JsonSerializer.SerializeToUtf8Bytes(new + { + Version = 2, + NextUpdate = "2024-11-02T00:00:00Z", + UpdatedAt = "2024-11-01T00:00:00Z", + }); + var builder = new RecordingTrivyDbBuilder(_root, builderMetadata); + var orasPusher = new StubTrivyDbOrasPusher(); + var exporter = new TrivyDbFeedExporter( + advisoryStore, + new VulnListJsonExportPathResolver(), + options, + packageBuilder, + ociWriter, + stateManager, + planner, + builder, + orasPusher, + NullLogger.Instance, + timeProvider); + + using var provider = new ServiceCollection().BuildServiceProvider(); + + var initialExportId = timeProvider.GetUtcNow().ToString(optionsValue.TagFormat, CultureInfo.InvariantCulture); + await exporter.ExportAsync(provider, CancellationToken.None); + + var initialLayout = Path.Combine(optionsValue.OutputRoot, initialExportId); + var initialMetadata = ReadMetadata(Path.Combine(initialLayout, "metadata.json")); + Assert.Equal("full", initialMetadata.Mode); + var initialManifestDigest = ReadManifestDigest(initialLayout); + + advisoryStore.SetAdvisories(firstDelta); + timeProvider.Advance(TimeSpan.FromMinutes(15)); + var deltaExportId = timeProvider.GetUtcNow().ToString(optionsValue.TagFormat, CultureInfo.InvariantCulture); + await exporter.ExportAsync(provider, CancellationToken.None); + + var deltaLayout = Path.Combine(optionsValue.OutputRoot, deltaExportId); + var deltaMetadata = ReadMetadata(Path.Combine(deltaLayout, "metadata.json")); + Assert.Equal("delta", deltaMetadata.Mode); + Assert.Equal(initialExportId, deltaMetadata.BaseExportId); + Assert.Equal(initialManifestDigest, deltaMetadata.BaseManifestDigest); + Assert.True(deltaMetadata.DeltaChangedCount > 0); + + var reusedManifestPath = Path.Combine(deltaLayout, "blobs", "sha256", initialManifestDigest[7..]); + Assert.True(File.Exists(reusedManifestPath)); + + advisoryStore.SetAdvisories(secondDelta); + timeProvider.Advance(TimeSpan.FromMinutes(15)); + var finalExportId = timeProvider.GetUtcNow().ToString(optionsValue.TagFormat, CultureInfo.InvariantCulture); + await exporter.ExportAsync(provider, CancellationToken.None); + + var finalLayout = Path.Combine(optionsValue.OutputRoot, finalExportId); + var finalMetadata = ReadMetadata(Path.Combine(finalLayout, "metadata.json")); + Assert.Equal("full", finalMetadata.Mode); + Assert.True(finalMetadata.ResetBaseline); + + var state = await stateStore.FindAsync(TrivyDbFeedExporter.ExporterId, CancellationToken.None); + Assert.NotNull(state); + Assert.Null(state!.LastDeltaDigest); + Assert.Equal(finalExportId, state.BaseExportId); + } + + [Fact] + public async Task ExportAsync_DeltaReusesBaseLayerOnDisk() + { + var baseline = CreateSampleAdvisory("CVE-2024-8300", "Layer reuse baseline"); + var delta = CreateSampleAdvisory("CVE-2024-8300", "Layer reuse delta"); + + var advisoryStore = new StubAdvisoryStore(baseline); + + var optionsValue = new TrivyDbExportOptions + { + OutputRoot = _root, + ReferencePrefix = "example/trivy", + KeepWorkingTree = true, + Json = new JsonExportOptions + { + OutputRoot = _jsonRoot, + MaintainLatestSymlink = false, + }, + }; + + var options = Options.Create(optionsValue); + var packageBuilder = new TrivyDbPackageBuilder(); + var ociWriter = new TrivyDbOciWriter(); + var planner = new TrivyDbExportPlanner(); + var stateStore = new InMemoryExportStateStore(); + var timeProvider = new TestTimeProvider(DateTimeOffset.Parse("2024-11-05T00:00:00Z", CultureInfo.InvariantCulture)); + var stateManager = new ExportStateManager(stateStore, timeProvider); + var builderMetadata = JsonSerializer.SerializeToUtf8Bytes(new + { + Version = 2, + NextUpdate = "2024-11-06T00:00:00Z", + UpdatedAt = "2024-11-05T00:00:00Z", + }); + var builder = new RecordingTrivyDbBuilder(_root, builderMetadata); + var orasPusher = new StubTrivyDbOrasPusher(); + var exporter = new TrivyDbFeedExporter( + advisoryStore, + new VulnListJsonExportPathResolver(), + options, + packageBuilder, + ociWriter, + stateManager, + planner, + builder, + orasPusher, + NullLogger.Instance, + timeProvider); + + using var provider = new ServiceCollection().BuildServiceProvider(); + + var baselineExportId = timeProvider.GetUtcNow().ToString(optionsValue.TagFormat, CultureInfo.InvariantCulture); + await exporter.ExportAsync(provider, CancellationToken.None); + + var baselineLayout = Path.Combine(optionsValue.OutputRoot, baselineExportId); + var baselineManifestDigest = ReadManifestDigest(baselineLayout); + var baselineLayerDigests = ReadManifestLayerDigests(baselineLayout, baselineManifestDigest); + var baselineLayerDigest = Assert.Single(baselineLayerDigests); + var baselineLayerPath = Path.Combine(baselineLayout, "blobs", "sha256", baselineLayerDigest[7..]); + var baselineLayerBytes = File.ReadAllBytes(baselineLayerPath); + + advisoryStore.SetAdvisories(delta); + timeProvider.Advance(TimeSpan.FromMinutes(30)); + var deltaExportId = timeProvider.GetUtcNow().ToString(optionsValue.TagFormat, CultureInfo.InvariantCulture); + await exporter.ExportAsync(provider, CancellationToken.None); + + var deltaLayout = Path.Combine(optionsValue.OutputRoot, deltaExportId); + var deltaMetadata = ReadMetadata(Path.Combine(deltaLayout, "metadata.json")); + Assert.Equal("delta", deltaMetadata.Mode); + Assert.Equal(baselineExportId, deltaMetadata.BaseExportId); + Assert.Equal(baselineManifestDigest, deltaMetadata.BaseManifestDigest); + Assert.True(deltaMetadata.DeltaChangedCount > 0); + + var deltaManifestDigest = ReadManifestDigest(deltaLayout); + Assert.NotEqual(baselineManifestDigest, deltaManifestDigest); + var deltaLayerDigests = ReadManifestLayerDigests(deltaLayout, deltaManifestDigest); + Assert.Contains(baselineLayerDigest, deltaLayerDigests); + + var deltaLayerPath = Path.Combine(deltaLayout, "blobs", "sha256", baselineLayerDigest[7..]); + Assert.True(File.Exists(deltaLayerPath)); + var deltaLayerBytes = File.ReadAllBytes(deltaLayerPath); + Assert.Equal(baselineLayerBytes, deltaLayerBytes); + } + + private static Advisory CreateSampleAdvisory( + string advisoryKey = "CVE-2024-9999", + string title = "Trivy Export Test") + { + return new Advisory( + advisoryKey: advisoryKey, + title: title, + summary: null, + language: "en", + published: DateTimeOffset.Parse("2024-08-01T00:00:00Z", CultureInfo.InvariantCulture), + modified: DateTimeOffset.Parse("2024-08-02T00:00:00Z", CultureInfo.InvariantCulture), + severity: "medium", + exploitKnown: false, + aliases: new[] { "CVE-2024-9999" }, + references: Array.Empty(), + affectedPackages: Array.Empty(), + cvssMetrics: Array.Empty(), + provenance: Array.Empty()); + } + + public void Dispose() + { + try + { + if (Directory.Exists(_root)) + { + Directory.Delete(_root, recursive: true); + } + } + catch + { + // best effort cleanup + } + } + + private sealed class StubAdvisoryStore : IAdvisoryStore + { + private IReadOnlyList _advisories; + + public StubAdvisoryStore(params Advisory[] advisories) + { + _advisories = advisories; + } + + public void SetAdvisories(params Advisory[] advisories) + { + _advisories = advisories; + } + + public Task> GetRecentAsync(int limit, CancellationToken cancellationToken) + => Task.FromResult(_advisories); + + public Task FindAsync(string advisoryKey, CancellationToken cancellationToken) + => Task.FromResult(_advisories.FirstOrDefault(a => a.AdvisoryKey == advisoryKey)); + + public Task UpsertAsync(Advisory advisory, CancellationToken cancellationToken) + => Task.CompletedTask; + + public IAsyncEnumerable StreamAsync(CancellationToken cancellationToken) + { + return EnumerateAsync(cancellationToken); + + async IAsyncEnumerable EnumerateAsync([EnumeratorCancellation] CancellationToken ct) + { + foreach (var advisory in _advisories) + { + ct.ThrowIfCancellationRequested(); + yield return advisory; + await Task.Yield(); + } + } + } + } + + private sealed class InMemoryExportStateStore : IExportStateStore + { + private ExportStateRecord? _record; + + public Task FindAsync(string id, CancellationToken cancellationToken) + => Task.FromResult(_record); + + public Task UpsertAsync(ExportStateRecord record, CancellationToken cancellationToken) + { + _record = record; + return Task.FromResult(record); + } + } + + private sealed class TestTimeProvider : TimeProvider + { + private DateTimeOffset _now; + + public TestTimeProvider(DateTimeOffset start) => _now = start; + + public override DateTimeOffset GetUtcNow() => _now; + + public void Advance(TimeSpan delta) => _now = _now.Add(delta); + } + + private sealed class StubTrivyDbBuilder : ITrivyDbBuilder + { + private readonly string _root; + private readonly byte[] _metadata; + + public StubTrivyDbBuilder(string root, byte[] metadata) + { + _root = root; + _metadata = metadata; + } + + public Task BuildAsync( + JsonExportResult jsonTree, + DateTimeOffset exportedAt, + string exportId, + CancellationToken cancellationToken) + { + var workingDirectory = Directory.CreateDirectory(Path.Combine(_root, $"builder-{exportId}")).FullName; + var archivePath = Path.Combine(workingDirectory, "db.tar.gz"); + var payload = new byte[] { 0x1, 0x2, 0x3, 0x4 }; + File.WriteAllBytes(archivePath, payload); + using var sha256 = SHA256.Create(); + var digest = "sha256:" + Convert.ToHexString(sha256.ComputeHash(payload)).ToLowerInvariant(); + var length = payload.Length; + + return Task.FromResult(new TrivyDbBuilderResult( + archivePath, + digest, + length, + _metadata, + workingDirectory)); + } + } + + private sealed class RecordingTrivyDbBuilder : ITrivyDbBuilder + { + private readonly string _root; + private readonly byte[] _metadata; + private readonly List _manifestDigests = new(); + + public RecordingTrivyDbBuilder(string root, byte[] metadata) + { + _root = root; + _metadata = metadata; + } + + public IReadOnlyList ManifestDigests => _manifestDigests; + public string[]? LastRelativePaths { get; private set; } + + public Task BuildAsync( + JsonExportResult jsonTree, + DateTimeOffset exportedAt, + string exportId, + CancellationToken cancellationToken) + { + LastRelativePaths = jsonTree.Files.Select(static file => file.RelativePath).ToArray(); + + var workingDirectory = Directory.CreateDirectory(Path.Combine(_root, $"builder-{exportId}")).FullName; + var archivePath = Path.Combine(workingDirectory, "db.tar.gz"); + var payload = new byte[] { 0x5, 0x6, 0x7, 0x8 }; + File.WriteAllBytes(archivePath, payload); + using var sha256 = SHA256.Create(); + var digest = "sha256:" + Convert.ToHexString(sha256.ComputeHash(payload)).ToLowerInvariant(); + _manifestDigests.Add(digest); + + return Task.FromResult(new TrivyDbBuilderResult( + archivePath, + digest, + payload.Length, + _metadata, + workingDirectory)); + } + } + + private sealed record MetadataView(string Mode, bool ResetBaseline, string? BaseExportId, string? BaseManifestDigest, int DeltaChangedCount); + + private static MetadataView ReadMetadata(string path) + { + using var document = JsonDocument.Parse(File.ReadAllText(path)); + var root = document.RootElement; + var mode = root.TryGetProperty("mode", out var modeNode) ? modeNode.GetString() ?? string.Empty : string.Empty; + var resetBaseline = root.TryGetProperty("resetBaseline", out var resetNode) && resetNode.ValueKind == JsonValueKind.True; + string? baseExportId = null; + if (root.TryGetProperty("baseExportId", out var baseExportNode) && baseExportNode.ValueKind == JsonValueKind.String) + { + baseExportId = baseExportNode.GetString(); + } + + string? baseManifestDigest = null; + if (root.TryGetProperty("baseManifestDigest", out var baseManifestNode) && baseManifestNode.ValueKind == JsonValueKind.String) + { + baseManifestDigest = baseManifestNode.GetString(); + } + + var deltaChangedCount = 0; + if (root.TryGetProperty("delta", out var deltaNode) && deltaNode.ValueKind == JsonValueKind.Object) + { + if (deltaNode.TryGetProperty("changedFiles", out var changedFilesNode) && changedFilesNode.ValueKind == JsonValueKind.Array) + { + deltaChangedCount = changedFilesNode.GetArrayLength(); + } + } + + return new MetadataView(mode, resetBaseline, baseExportId, baseManifestDigest, deltaChangedCount); + } + + private static string ReadManifestDigest(string layoutPath) + { + var indexPath = Path.Combine(layoutPath, "index.json"); + using var document = JsonDocument.Parse(File.ReadAllText(indexPath)); + var manifests = document.RootElement.GetProperty("manifests"); + if (manifests.GetArrayLength() == 0) + { + throw new InvalidOperationException("No manifests present in OCI index."); + } + + return manifests[0].GetProperty("digest").GetString() ?? string.Empty; + } + + private static string[] ReadManifestLayerDigests(string layoutPath, string manifestDigest) + { + var manifestPath = Path.Combine(layoutPath, "blobs", "sha256", manifestDigest[7..]); + using var document = JsonDocument.Parse(File.ReadAllText(manifestPath)); + var layers = document.RootElement.GetProperty("layers"); + var digests = new string[layers.GetArrayLength()]; + var index = 0; + foreach (var layer in layers.EnumerateArray()) + { + digests[index++] = layer.GetProperty("digest").GetString() ?? string.Empty; + } + + return digests; + } + + private sealed record RunArtifacts( + string ExportId, + string ManifestDigest, + string IndexJson, + string MetadataJson, + string ManifestJson, + IReadOnlyDictionary Blobs); + + private async Task RunDeterministicExportAsync(IReadOnlyList advisories) + { + var workspace = Path.Combine(_root, $"deterministic-{Guid.NewGuid():N}"); + var jsonRoot = Path.Combine(workspace, "tree"); + Directory.CreateDirectory(workspace); + + var advisoryStore = new StubAdvisoryStore(advisories.ToArray()); + + var optionsValue = new TrivyDbExportOptions + { + OutputRoot = workspace, + ReferencePrefix = "example/trivy", + KeepWorkingTree = true, + Json = new JsonExportOptions + { + OutputRoot = jsonRoot, + MaintainLatestSymlink = false, + }, + }; + + var exportedAt = DateTimeOffset.Parse("2024-10-01T00:00:00Z", CultureInfo.InvariantCulture); + var options = Options.Create(optionsValue); + var packageBuilder = new TrivyDbPackageBuilder(); + var ociWriter = new TrivyDbOciWriter(); + var planner = new TrivyDbExportPlanner(); + var stateStore = new InMemoryExportStateStore(); + var timeProvider = new TestTimeProvider(exportedAt); + var stateManager = new ExportStateManager(stateStore, timeProvider); + var builderMetadata = JsonSerializer.SerializeToUtf8Bytes(new + { + Version = 2, + NextUpdate = "2024-10-02T00:00:00Z", + UpdatedAt = "2024-10-01T00:00:00Z", + }); + + var builder = new DeterministicTrivyDbBuilder(workspace, builderMetadata); + var orasPusher = new StubTrivyDbOrasPusher(); + var exporter = new TrivyDbFeedExporter( + advisoryStore, + new VulnListJsonExportPathResolver(), + options, + packageBuilder, + ociWriter, + stateManager, + planner, + builder, + orasPusher, + NullLogger.Instance, + timeProvider); + + using var provider = new ServiceCollection().BuildServiceProvider(); + await exporter.ExportAsync(provider, CancellationToken.None); + + var exportId = exportedAt.ToString(optionsValue.TagFormat, CultureInfo.InvariantCulture); + var layoutPath = Path.Combine(workspace, exportId); + + var indexJson = await File.ReadAllTextAsync(Path.Combine(layoutPath, "index.json"), Encoding.UTF8); + var metadataJson = await File.ReadAllTextAsync(Path.Combine(layoutPath, "metadata.json"), Encoding.UTF8); + + using var indexDoc = JsonDocument.Parse(indexJson); + var manifestNode = indexDoc.RootElement.GetProperty("manifests")[0]; + var manifestDigest = manifestNode.GetProperty("digest").GetString()!; + + var manifestHex = manifestDigest[7..]; + var manifestJson = await File.ReadAllTextAsync(Path.Combine(layoutPath, "blobs", "sha256", manifestHex), Encoding.UTF8); + + var blobs = new Dictionary(StringComparer.Ordinal); + var blobsRoot = Path.Combine(layoutPath, "blobs", "sha256"); + foreach (var file in Directory.GetFiles(blobsRoot)) + { + var name = Path.GetFileName(file); + var content = await File.ReadAllBytesAsync(file); + blobs[name] = content; + } + + Directory.Delete(workspace, recursive: true); + + return new RunArtifacts(exportId, manifestDigest, indexJson, metadataJson, manifestJson, blobs); + } + + private sealed class DeterministicTrivyDbBuilder : ITrivyDbBuilder + { + private readonly string _root; + private readonly byte[] _metadata; + private readonly byte[] _payload; + + public DeterministicTrivyDbBuilder(string root, byte[] metadata) + { + _root = root; + _metadata = metadata; + _payload = new byte[] { 0x21, 0x22, 0x23, 0x24, 0x25 }; + } + + public Task BuildAsync( + JsonExportResult jsonTree, + DateTimeOffset exportedAt, + string exportId, + CancellationToken cancellationToken) + { + var workingDirectory = Directory.CreateDirectory(Path.Combine(_root, $"builder-{exportId}")).FullName; + var archivePath = Path.Combine(workingDirectory, "db.tar.gz"); + File.WriteAllBytes(archivePath, _payload); + using var sha256 = SHA256.Create(); + var digest = "sha256:" + Convert.ToHexString(sha256.ComputeHash(_payload)).ToLowerInvariant(); + + return Task.FromResult(new TrivyDbBuilderResult( + archivePath, + digest, + _payload.Length, + _metadata, + workingDirectory)); + } + } + + private sealed class StubTrivyDbOrasPusher : ITrivyDbOrasPusher + { + public List<(string Layout, string Reference, string ExportId)> Pushes { get; } = new(); + + public Task PushAsync(string layoutPath, string reference, string exportId, CancellationToken cancellationToken) + { + Pushes.Add((layoutPath, reference, exportId)); + return Task.CompletedTask; + } + } +} diff --git a/src/StellaOps.Feedser.Exporter.TrivyDb.Tests/TrivyDbOciWriterTests.cs b/src/StellaOps.Feedser.Exporter.TrivyDb.Tests/TrivyDbOciWriterTests.cs new file mode 100644 index 00000000..64ae7f0b --- /dev/null +++ b/src/StellaOps.Feedser.Exporter.TrivyDb.Tests/TrivyDbOciWriterTests.cs @@ -0,0 +1,149 @@ +using System; +using System.Collections.Generic; +using System.IO; +using System.Reflection; +using System.Security.Cryptography; +using System.Text; +using System.Text.Json; +using System.Threading; +using System.Threading.Tasks; +using StellaOps.Feedser.Storage.Mongo.Exporting; +using Xunit; + +namespace StellaOps.Feedser.Exporter.TrivyDb.Tests; + +public sealed class TrivyDbOciWriterTests : IDisposable +{ + private readonly string _root; + + public TrivyDbOciWriterTests() + { + _root = Directory.CreateTempSubdirectory("trivy-writer-tests").FullName; + } + + [Fact] + public async Task WriteAsync_ReusesBlobsFromBaseLayout_WhenDigestMatches() + { + var baseLayout = Path.Combine(_root, "base"); + Directory.CreateDirectory(Path.Combine(baseLayout, "blobs", "sha256")); + + var configBytes = Encoding.UTF8.GetBytes("base-config"); + var configDigest = ComputeDigest(configBytes); + WriteBlob(baseLayout, configDigest, configBytes); + + var layerBytes = Encoding.UTF8.GetBytes("base-layer"); + var layerDigest = ComputeDigest(layerBytes); + WriteBlob(baseLayout, layerDigest, layerBytes); + + var manifest = CreateManifest(configDigest, layerDigest); + var manifestBytes = SerializeManifest(manifest); + var manifestDigest = ComputeDigest(manifestBytes); + WriteBlob(baseLayout, manifestDigest, manifestBytes); + + var plan = new TrivyDbExportPlan( + TrivyDbExportMode.Delta, + TreeDigest: "sha256:tree", + BaseExportId: "20241101T000000Z", + BaseManifestDigest: manifestDigest, + ResetBaseline: false, + Manifest: Array.Empty(), + ChangedFiles: new[] { new ExportFileRecord("data.json", 1, "sha256:data") }, + RemovedPaths: Array.Empty()); + + var configDescriptor = new OciDescriptor(TrivyDbMediaTypes.TrivyConfig, configDigest, configBytes.Length); + var layerDescriptor = new OciDescriptor(TrivyDbMediaTypes.TrivyLayer, layerDigest, layerBytes.Length); + var package = new TrivyDbPackage( + manifest, + new TrivyConfigDocument( + TrivyDbMediaTypes.TrivyConfig, + DateTimeOffset.Parse("2024-11-01T00:00:00Z"), + "20241101T000000Z", + layerDigest, + layerBytes.Length), + new Dictionary(StringComparer.Ordinal) + { + [configDigest] = CreateThrowingBlob(), + [layerDigest] = CreateThrowingBlob(), + }, + JsonSerializer.SerializeToUtf8Bytes(new { mode = "delta" })); + + var writer = new TrivyDbOciWriter(); + var destination = Path.Combine(_root, "delta"); + await writer.WriteAsync(package, destination, reference: "example/trivy:delta", plan, baseLayout, CancellationToken.None); + + var reusedConfig = File.ReadAllBytes(GetBlobPath(destination, configDigest)); + Assert.Equal(configBytes, reusedConfig); + + var reusedLayer = File.ReadAllBytes(GetBlobPath(destination, layerDigest)); + Assert.Equal(layerBytes, reusedLayer); + } + + private static TrivyDbBlob CreateThrowingBlob() + { + var ctor = typeof(TrivyDbBlob).GetConstructor( + BindingFlags.NonPublic | BindingFlags.Instance, + binder: null, + new[] { typeof(Func>), typeof(long) }, + modifiers: null) + ?? throw new InvalidOperationException("Unable to access TrivyDbBlob constructor."); + + Func> factory = _ => throw new InvalidOperationException("Blob should have been reused from base layout."); + return (TrivyDbBlob)ctor.Invoke(new object[] { factory, 0L }); + } + + private static OciManifest CreateManifest(string configDigest, string layerDigest) + { + var configDescriptor = new OciDescriptor(TrivyDbMediaTypes.TrivyConfig, configDigest, 0); + var layerDescriptor = new OciDescriptor(TrivyDbMediaTypes.TrivyLayer, layerDigest, 0); + return new OciManifest( + SchemaVersion: 2, + MediaType: TrivyDbMediaTypes.OciManifest, + Config: configDescriptor, + Layers: new[] { layerDescriptor }); + } + + private static byte[] SerializeManifest(OciManifest manifest) + { + var options = new JsonSerializerOptions + { + PropertyNamingPolicy = JsonNamingPolicy.CamelCase, + DefaultIgnoreCondition = System.Text.Json.Serialization.JsonIgnoreCondition.WhenWritingNull, + WriteIndented = false, + }; + return JsonSerializer.SerializeToUtf8Bytes(manifest, options); + } + + private static void WriteBlob(string layoutRoot, string digest, byte[] payload) + { + var path = GetBlobPath(layoutRoot, digest); + Directory.CreateDirectory(Path.GetDirectoryName(path)!); + File.WriteAllBytes(path, payload); + } + + private static string GetBlobPath(string layoutRoot, string digest) + { + var fileName = digest[7..]; + return Path.Combine(layoutRoot, "blobs", "sha256", fileName); + } + + private static string ComputeDigest(byte[] payload) + { + var hash = SHA256.HashData(payload); + return "sha256:" + Convert.ToHexString(hash).ToLowerInvariant(); + } + + public void Dispose() + { + try + { + if (Directory.Exists(_root)) + { + Directory.Delete(_root, recursive: true); + } + } + catch + { + // best effort cleanup + } + } +} diff --git a/src/StellaOps.Feedser.Exporter.TrivyDb.Tests/TrivyDbPackageBuilderTests.cs b/src/StellaOps.Feedser.Exporter.TrivyDb.Tests/TrivyDbPackageBuilderTests.cs new file mode 100644 index 00000000..dcd4a6ab --- /dev/null +++ b/src/StellaOps.Feedser.Exporter.TrivyDb.Tests/TrivyDbPackageBuilderTests.cs @@ -0,0 +1,93 @@ +using System; +using System.IO; +using System.Linq; +using System.Security.Cryptography; +using System.Text; +using System.Text.Json; +using StellaOps.Feedser.Exporter.TrivyDb; + +namespace StellaOps.Feedser.Exporter.TrivyDb.Tests; + +public sealed class TrivyDbPackageBuilderTests +{ + [Fact] + public void BuildsOciManifestWithExpectedMediaTypes() + { + var metadata = Encoding.UTF8.GetBytes("{\"generatedAt\":\"2024-07-15T12:00:00Z\"}"); + var archive = Enumerable.Range(0, 256).Select(static b => (byte)b).ToArray(); + var archivePath = Path.GetTempFileName(); + File.WriteAllBytes(archivePath, archive); + var archiveDigest = ComputeDigest(archive); + + try + { + var request = new TrivyDbPackageRequest( + metadata, + archivePath, + archiveDigest, + archive.LongLength, + DateTimeOffset.Parse("2024-07-15T12:00:00Z"), + "2024.07.15"); + + var builder = new TrivyDbPackageBuilder(); + var package = builder.BuildPackage(request); + + Assert.Equal(TrivyDbMediaTypes.OciManifest, package.Manifest.MediaType); + Assert.Equal(TrivyDbMediaTypes.TrivyConfig, package.Manifest.Config.MediaType); + var layer = Assert.Single(package.Manifest.Layers); + Assert.Equal(TrivyDbMediaTypes.TrivyLayer, layer.MediaType); + + var configBytes = JsonSerializer.SerializeToUtf8Bytes(package.Config, new JsonSerializerOptions { PropertyNamingPolicy = JsonNamingPolicy.CamelCase }); + var expectedConfigDigest = ComputeDigest(configBytes); + Assert.Equal(expectedConfigDigest, package.Manifest.Config.Digest); + + Assert.Equal(archiveDigest, layer.Digest); + Assert.True(package.Blobs.ContainsKey(archiveDigest)); + Assert.Equal(archive.LongLength, package.Blobs[archiveDigest].Length); + Assert.True(package.Blobs.ContainsKey(expectedConfigDigest)); + Assert.Equal(metadata, package.MetadataJson.ToArray()); + } + finally + { + if (File.Exists(archivePath)) + { + File.Delete(archivePath); + } + } + } + + [Fact] + public void ThrowsWhenMetadataMissing() + { + var builder = new TrivyDbPackageBuilder(); + var archivePath = Path.GetTempFileName(); + var archiveBytes = new byte[] { 1, 2, 3 }; + File.WriteAllBytes(archivePath, archiveBytes); + var digest = ComputeDigest(archiveBytes); + + try + { + Assert.Throws(() => builder.BuildPackage(new TrivyDbPackageRequest( + ReadOnlyMemory.Empty, + archivePath, + digest, + archiveBytes.LongLength, + DateTimeOffset.UtcNow, + "1"))); + } + finally + { + if (File.Exists(archivePath)) + { + File.Delete(archivePath); + } + } + } + + private static string ComputeDigest(ReadOnlySpan payload) + { + var hash = SHA256.HashData(payload); + var hex = Convert.ToHexString(hash); + return "sha256:" + hex.ToLowerInvariant(); + } +} diff --git a/src/StellaOps.Feedser.Exporter.TrivyDb/AGENTS.md b/src/StellaOps.Feedser.Exporter.TrivyDb/AGENTS.md new file mode 100644 index 00000000..2897733c --- /dev/null +++ b/src/StellaOps.Feedser.Exporter.TrivyDb/AGENTS.md @@ -0,0 +1,29 @@ +# AGENTS +## Role +Exporter producing a Trivy-compatible database artifact for self-hosting or offline use. v0: JSON list + metadata; v1: integrate official trivy-db builder or write BoltDB directly; pack and optionally push via ORAS. +## Scope +- Read canonical advisories; serialize payload for builder or intermediate; write metadata.json (generatedAt, counts). +- Output root: exports/trivy/; deterministic path components. +- OCI/Trivy expectations: layer media type application/vnd.aquasec.trivy.db.layer.v1.tar+gzip; config media type application/vnd.aquasec.trivy.config.v1+json; tag (e.g., 2). +- Optional ORAS push; optional offline bundle (db.tar.gz + metadata.json). +- DI: TrivyExporter + Jobs.TrivyExportJob registered by TrivyExporterDependencyInjectionRoutine. +- Export_state recording: capture digests, counts, start/end timestamps for idempotent reruns and incremental packaging. +## Participants +- Storage.Mongo.AdvisoryStore as input. +- Core scheduler runs export job; WebService/Plugins trigger it. +- JSON exporter (optional precursor) if choosing the builder path. +## Interfaces & contracts +- IFeedExporter.Name = "trivy-db"; ExportAsync(IServiceProvider, CancellationToken). +- FeedserOptions.packaging.trivy governs repo/tag/publish/offline_bundle. +- Deterministic sorting and timestamp discipline (UTC; consider build reproducibility knobs). +## In/Out of scope +In: assembling builder inputs, packing tar.gz, pushing to registry when configured. +Out: signing (external pipeline), scanner behavior. +## Observability & security expectations +- Metrics: export.trivy.records, size_bytes, duration, oras.push.success/fail. +- Logs: export path, repo/tag, digest; redact credentials; backoff on push errors. +## Tests +- Author and review coverage in `../StellaOps.Feedser.Exporter.TrivyDb.Tests`. +- Shared fixtures (e.g., `MongoIntegrationFixture`, `ConnectorTestHarness`) live in `../StellaOps.Feedser.Testing`. +- Keep fixtures deterministic; match new cases to real-world advisories or regression scenarios. + diff --git a/src/StellaOps.Feedser.Exporter.TrivyDb/ITrivyDbBuilder.cs b/src/StellaOps.Feedser.Exporter.TrivyDb/ITrivyDbBuilder.cs new file mode 100644 index 00000000..0f9854ba --- /dev/null +++ b/src/StellaOps.Feedser.Exporter.TrivyDb/ITrivyDbBuilder.cs @@ -0,0 +1,15 @@ +using System; +using System.Threading; +using System.Threading.Tasks; +using StellaOps.Feedser.Exporter.Json; + +namespace StellaOps.Feedser.Exporter.TrivyDb; + +public interface ITrivyDbBuilder +{ + Task BuildAsync( + JsonExportResult jsonTree, + DateTimeOffset exportedAt, + string exportId, + CancellationToken cancellationToken); +} diff --git a/src/StellaOps.Feedser.Exporter.TrivyDb/ITrivyDbOrasPusher.cs b/src/StellaOps.Feedser.Exporter.TrivyDb/ITrivyDbOrasPusher.cs new file mode 100644 index 00000000..d8a44075 --- /dev/null +++ b/src/StellaOps.Feedser.Exporter.TrivyDb/ITrivyDbOrasPusher.cs @@ -0,0 +1,9 @@ +using System.Threading; +using System.Threading.Tasks; + +namespace StellaOps.Feedser.Exporter.TrivyDb; + +public interface ITrivyDbOrasPusher +{ + Task PushAsync(string layoutPath, string reference, string exportId, CancellationToken cancellationToken); +} diff --git a/src/StellaOps.Feedser.Exporter.TrivyDb/OciDescriptor.cs b/src/StellaOps.Feedser.Exporter.TrivyDb/OciDescriptor.cs new file mode 100644 index 00000000..58aeaec8 --- /dev/null +++ b/src/StellaOps.Feedser.Exporter.TrivyDb/OciDescriptor.cs @@ -0,0 +1,10 @@ +using System.Collections.Generic; +using System.Text.Json.Serialization; + +namespace StellaOps.Feedser.Exporter.TrivyDb; + +public sealed record OciDescriptor( + [property: JsonPropertyName("mediaType")] string MediaType, + [property: JsonPropertyName("digest")] string Digest, + [property: JsonPropertyName("size")] long Size, + [property: JsonPropertyName("annotations")] IReadOnlyDictionary? Annotations = null); diff --git a/src/StellaOps.Feedser.Exporter.TrivyDb/OciIndex.cs b/src/StellaOps.Feedser.Exporter.TrivyDb/OciIndex.cs new file mode 100644 index 00000000..eb00c179 --- /dev/null +++ b/src/StellaOps.Feedser.Exporter.TrivyDb/OciIndex.cs @@ -0,0 +1,8 @@ +using System.Collections.Generic; +using System.Text.Json.Serialization; + +namespace StellaOps.Feedser.Exporter.TrivyDb; + +public sealed record OciIndex( + [property: JsonPropertyName("schemaVersion")] int SchemaVersion, + [property: JsonPropertyName("manifests")] IReadOnlyList Manifests); diff --git a/src/StellaOps.Feedser.Exporter.TrivyDb/OciManifest.cs b/src/StellaOps.Feedser.Exporter.TrivyDb/OciManifest.cs new file mode 100644 index 00000000..ee99d638 --- /dev/null +++ b/src/StellaOps.Feedser.Exporter.TrivyDb/OciManifest.cs @@ -0,0 +1,10 @@ +using System.Collections.Generic; +using System.Text.Json.Serialization; + +namespace StellaOps.Feedser.Exporter.TrivyDb; + +public sealed record OciManifest( + [property: JsonPropertyName("schemaVersion")] int SchemaVersion, + [property: JsonPropertyName("mediaType")] string MediaType, + [property: JsonPropertyName("config")] OciDescriptor Config, + [property: JsonPropertyName("layers")] IReadOnlyList Layers); diff --git a/src/StellaOps.Feedser.Exporter.TrivyDb/StellaOps.Feedser.Exporter.TrivyDb.csproj b/src/StellaOps.Feedser.Exporter.TrivyDb/StellaOps.Feedser.Exporter.TrivyDb.csproj new file mode 100644 index 00000000..fe28b621 --- /dev/null +++ b/src/StellaOps.Feedser.Exporter.TrivyDb/StellaOps.Feedser.Exporter.TrivyDb.csproj @@ -0,0 +1,22 @@ + + + + net10.0 + preview + enable + enable + true + + + + + + + + + + + + + + \ No newline at end of file diff --git a/src/StellaOps.Feedser.Exporter.TrivyDb/TASKS.md b/src/StellaOps.Feedser.Exporter.TrivyDb/TASKS.md new file mode 100644 index 00000000..cd0d0120 --- /dev/null +++ b/src/StellaOps.Feedser.Exporter.TrivyDb/TASKS.md @@ -0,0 +1,13 @@ +# TASKS +| Task | Owner(s) | Depends on | Notes | +|---|---|---|---| +|Fix method name typo GetExportRoot' -> GetExportRoot|BE-Export|Exporters|DONE – `TrivyDbExportOptions.GetExportRoot` helper added with unit coverage.| +|Implement BoltDB builder integration (v0 via trivy-db CLI)|BE-Export|Env|DONE – `TrivyDbBoltBuilder` shells `trivy-db build` against our JSON tree with deterministic packaging.| +|Pack db.tar.gz + metadata.json|BE-Export|Exporters|DONE – Builder output re-packed with fixed timestamps and zeroed gzip mtime.| +|ORAS push support|BE-Export|Exporters|DONE – Optional `TrivyDbOrasPusher` shells `oras cp --from-oci-layout` with configurable args/env.| +|Offline bundle toggle|BE-Export|Exporters|DONE – Deterministic OCI layout bundle emitted when enabled.| +|Deterministic ordering of advisories|BE-Export|Models|DONE – exporter now loads advisories, sorts by advisoryKey, and emits sorted JSON trees with deterministic OCI payloads.| +|End-to-end tests with small dataset|QA|Exporters|DONE – added deterministic round-trip test covering OCI layout, media types, and digest stability w/ repeated inputs.| +|ExportState persistence & idempotence|BE-Export|Storage.Mongo|DONE – baseline resets wired into `ExportStateManager`, planner signals resets after delta runs, and exporters update state w/ repository-aware baseline rotation + tests.| +|Streamed package building to avoid large copies|BE-Export|Exporters|DONE – metadata/config now reuse backing arrays and OCI writer streams directly without double buffering.| +|Plan incremental/delta exports|BE-Export|Exporters|DONE – state captures per-file manifests, planner schedules delta vs full resets, layer reuse smoke test verifies OCI reuse, and operator guide documents the validation flow.| diff --git a/src/StellaOps.Feedser.Exporter.TrivyDb/TrivyConfigDocument.cs b/src/StellaOps.Feedser.Exporter.TrivyDb/TrivyConfigDocument.cs new file mode 100644 index 00000000..10e947c8 --- /dev/null +++ b/src/StellaOps.Feedser.Exporter.TrivyDb/TrivyConfigDocument.cs @@ -0,0 +1,11 @@ +using System; +using System.Text.Json.Serialization; + +namespace StellaOps.Feedser.Exporter.TrivyDb; + +public sealed record TrivyConfigDocument( + [property: JsonPropertyName("mediaType")] string MediaType, + [property: JsonPropertyName("generatedAt")] DateTimeOffset GeneratedAt, + [property: JsonPropertyName("databaseVersion")] string DatabaseVersion, + [property: JsonPropertyName("databaseDigest")] string DatabaseDigest, + [property: JsonPropertyName("databaseSize")] long DatabaseSize); diff --git a/src/StellaOps.Feedser.Exporter.TrivyDb/TrivyDbBlob.cs b/src/StellaOps.Feedser.Exporter.TrivyDb/TrivyDbBlob.cs new file mode 100644 index 00000000..2ff58e60 --- /dev/null +++ b/src/StellaOps.Feedser.Exporter.TrivyDb/TrivyDbBlob.cs @@ -0,0 +1,78 @@ +using System; +using System.IO; +using System.Runtime.InteropServices; +using System.Threading; +using System.Threading.Tasks; + +namespace StellaOps.Feedser.Exporter.TrivyDb; + +public sealed class TrivyDbBlob +{ + private readonly Func> _openReadAsync; + + private TrivyDbBlob(Func> openReadAsync, long length) + { + _openReadAsync = openReadAsync ?? throw new ArgumentNullException(nameof(openReadAsync)); + if (length < 0) + { + throw new ArgumentOutOfRangeException(nameof(length)); + } + + Length = length; + } + + public long Length { get; } + + public ValueTask OpenReadAsync(CancellationToken cancellationToken) + => _openReadAsync(cancellationToken); + + public static TrivyDbBlob FromBytes(ReadOnlyMemory payload) + { + if (payload.IsEmpty) + { + return new TrivyDbBlob(static _ => ValueTask.FromResult(Stream.Null), 0); + } + + if (MemoryMarshal.TryGetArray(payload, out ArraySegment segment) && segment.Array is not null && segment.Offset == 0) + { + return FromArray(segment.Array); + } + + return FromArray(payload.ToArray()); + } + + public static TrivyDbBlob FromFile(string path, long length) + { + if (string.IsNullOrWhiteSpace(path)) + { + throw new ArgumentException("File path must be provided.", nameof(path)); + } + + if (length < 0) + { + throw new ArgumentOutOfRangeException(nameof(length)); + } + + return new TrivyDbBlob( + cancellationToken => ValueTask.FromResult(new FileStream( + path, + FileMode.Open, + FileAccess.Read, + FileShare.Read, + bufferSize: 81920, + options: FileOptions.Asynchronous | FileOptions.SequentialScan)), + length); + } + + public static TrivyDbBlob FromArray(byte[] buffer) + { + if (buffer is null) + { + throw new ArgumentNullException(nameof(buffer)); + } + + return new TrivyDbBlob( + _ => ValueTask.FromResult(new MemoryStream(buffer, writable: false)), + buffer.LongLength); + } +} diff --git a/src/StellaOps.Feedser.Exporter.TrivyDb/TrivyDbBoltBuilder.cs b/src/StellaOps.Feedser.Exporter.TrivyDb/TrivyDbBoltBuilder.cs new file mode 100644 index 00000000..e7723005 --- /dev/null +++ b/src/StellaOps.Feedser.Exporter.TrivyDb/TrivyDbBoltBuilder.cs @@ -0,0 +1,376 @@ +using System; +using System.Diagnostics; +using System.Globalization; +using System.IO; +using System.IO.Compression; +using System.Linq; +using System.Security.Cryptography; +using System.Text; +using System.Threading; +using System.Threading.Tasks; +using System.Formats.Tar; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Options; +using StellaOps.Feedser.Exporter.Json; + +namespace StellaOps.Feedser.Exporter.TrivyDb; + +public sealed class TrivyDbBoltBuilder : ITrivyDbBuilder +{ + private readonly TrivyDbExportOptions _options; + private readonly ILogger _logger; + + public TrivyDbBoltBuilder(IOptions options, ILogger logger) + { + _options = options?.Value ?? throw new ArgumentNullException(nameof(options)); + _logger = logger ?? throw new ArgumentNullException(nameof(logger)); + } + + public async Task BuildAsync( + JsonExportResult jsonTree, + DateTimeOffset exportedAt, + string exportId, + CancellationToken cancellationToken) + { + ArgumentNullException.ThrowIfNull(jsonTree); + ArgumentException.ThrowIfNullOrEmpty(exportId); + + var builderRoot = PrepareBuilderRoot(jsonTree.ExportDirectory, exportId); + var outputDir = Path.Combine(builderRoot, "out"); + Directory.CreateDirectory(outputDir); + + try + { + await RunCliAsync(jsonTree.ExportDirectory, outputDir, cancellationToken).ConfigureAwait(false); + } + catch + { + TryDeleteDirectory(builderRoot); + throw; + } + + var metadataPath = Path.Combine(outputDir, "metadata.json"); + var dbPath = Path.Combine(outputDir, "trivy.db"); + + if (!File.Exists(metadataPath)) + { + TryDeleteDirectory(builderRoot); + throw new InvalidOperationException($"trivy-db metadata not found at '{metadataPath}'."); + } + + if (!File.Exists(dbPath)) + { + TryDeleteDirectory(builderRoot); + throw new InvalidOperationException($"trivy.db not found at '{dbPath}'."); + } + + var archivePath = Path.Combine(builderRoot, "db.tar.gz"); + await CreateArchiveAsync(archivePath, exportedAt, metadataPath, dbPath, cancellationToken).ConfigureAwait(false); + + var digest = await ComputeDigestAsync(archivePath, cancellationToken).ConfigureAwait(false); + var length = new FileInfo(archivePath).Length; + var builderMetadata = await File.ReadAllBytesAsync(metadataPath, cancellationToken).ConfigureAwait(false); + + return new TrivyDbBuilderResult( + archivePath, + digest, + length, + builderMetadata, + builderRoot); + } + + private string PrepareBuilderRoot(string exportDirectory, string exportId) + { + var root = Path.Combine(exportDirectory, $".builder-{exportId}"); + if (Directory.Exists(root)) + { + Directory.Delete(root, recursive: true); + } + + Directory.CreateDirectory(root); + return root; + } + + private static void TryDeleteDirectory(string directory) + { + try + { + if (Directory.Exists(directory)) + { + Directory.Delete(directory, recursive: true); + } + } + catch + { + // ignore cleanup failures + } + } + + private async Task RunCliAsync(string cacheDir, string outputDir, CancellationToken cancellationToken) + { + var builderOptions = _options.Builder ?? new TrivyDbBuilderOptions(); + var executable = string.IsNullOrWhiteSpace(builderOptions.ExecutablePath) + ? "trivy-db" + : builderOptions.ExecutablePath; + + var targets = builderOptions.OnlyUpdateTargets ?? new System.Collections.Generic.List(); + var environment = builderOptions.Environment ?? new System.Collections.Generic.Dictionary(StringComparer.OrdinalIgnoreCase); + + var startInfo = new ProcessStartInfo + { + FileName = executable, + RedirectStandardOutput = true, + RedirectStandardError = true, + UseShellExecute = false, + }; + + startInfo.ArgumentList.Add("build"); + startInfo.ArgumentList.Add("--cache-dir"); + startInfo.ArgumentList.Add(cacheDir); + startInfo.ArgumentList.Add("--output-dir"); + startInfo.ArgumentList.Add(outputDir); + + if (builderOptions.UpdateInterval != default) + { + startInfo.ArgumentList.Add("--update-interval"); + startInfo.ArgumentList.Add(ToGoDuration(builderOptions.UpdateInterval)); + } + + if (targets.Count > 0) + { + foreach (var target in targets.Where(static t => !string.IsNullOrWhiteSpace(t))) + { + startInfo.ArgumentList.Add("--only-update"); + startInfo.ArgumentList.Add(target); + } + } + + if (!string.IsNullOrWhiteSpace(builderOptions.WorkingDirectory)) + { + startInfo.WorkingDirectory = builderOptions.WorkingDirectory; + } + + if (!builderOptions.InheritEnvironment) + { + startInfo.Environment.Clear(); + } + + foreach (var kvp in environment) + { + startInfo.Environment[kvp.Key] = kvp.Value; + } + + using var process = new Process { StartInfo = startInfo, EnableRaisingEvents = false }; + + var stdOut = new StringBuilder(); + var stdErr = new StringBuilder(); + + var stdoutCompletion = new TaskCompletionSource(); + var stderrCompletion = new TaskCompletionSource(); + + process.OutputDataReceived += (_, e) => + { + if (e.Data is null) + { + stdoutCompletion.TrySetResult(null); + } + else + { + stdOut.AppendLine(e.Data); + } + }; + + process.ErrorDataReceived += (_, e) => + { + if (e.Data is null) + { + stderrCompletion.TrySetResult(null); + } + else + { + stdErr.AppendLine(e.Data); + } + }; + + _logger.LogInformation("Running {Executable} to build Trivy DB", executable); + + try + { + if (!process.Start()) + { + throw new InvalidOperationException($"Failed to start '{executable}'."); + } + } + catch (Exception ex) + { + throw new InvalidOperationException($"Failed to start '{executable}'.", ex); + } + + process.BeginOutputReadLine(); + process.BeginErrorReadLine(); + + using var registration = cancellationToken.Register(() => + { + try + { + if (!process.HasExited) + { + process.Kill(entireProcessTree: true); + } + } + catch + { + // Ignore kill failures. + } + }); + +#if NET8_0_OR_GREATER + await process.WaitForExitAsync(cancellationToken).ConfigureAwait(false); +#else + await Task.Run(() => process.WaitForExit(), cancellationToken).ConfigureAwait(false); +#endif + + await Task.WhenAll(stdoutCompletion.Task, stderrCompletion.Task).ConfigureAwait(false); + + if (process.ExitCode != 0) + { + _logger.LogError("trivy-db exited with code {ExitCode}. stderr: {Stderr}", process.ExitCode, stdErr.ToString()); + throw new InvalidOperationException($"'{executable}' exited with code {process.ExitCode}."); + } + + if (stdOut.Length > 0) + { + _logger.LogDebug("trivy-db output: {StdOut}", stdOut.ToString()); + } + + if (stdErr.Length > 0) + { + _logger.LogWarning("trivy-db warnings: {StdErr}", stdErr.ToString()); + } + } + + private static async Task CreateArchiveAsync( + string archivePath, + DateTimeOffset exportedAt, + string metadataPath, + string dbPath, + CancellationToken cancellationToken) + { + await using var archiveStream = new FileStream( + archivePath, + FileMode.Create, + FileAccess.Write, + FileShare.None, + bufferSize: 81920, + options: FileOptions.Asynchronous | FileOptions.SequentialScan); + await using var gzip = new GZipStream(archiveStream, CompressionLevel.SmallestSize, leaveOpen: true); + await using var writer = new TarWriter(gzip, TarEntryFormat.Pax, leaveOpen: false); + + var timestamp = exportedAt.UtcDateTime; + foreach (var file in EnumerateArchiveEntries(metadataPath, dbPath)) + { + cancellationToken.ThrowIfCancellationRequested(); + + var entry = new PaxTarEntry(TarEntryType.RegularFile, file.Name) + { + ModificationTime = timestamp, + Mode = UnixFileMode.UserRead | UnixFileMode.UserWrite | UnixFileMode.GroupRead | UnixFileMode.OtherRead, + }; + + await using var source = new FileStream( + file.Path, + FileMode.Open, + FileAccess.Read, + FileShare.Read, + bufferSize: 81920, + options: FileOptions.Asynchronous | FileOptions.SequentialScan); + entry.DataStream = source; + writer.WriteEntry(entry); + } + + await writer.DisposeAsync().ConfigureAwait(false); + await ZeroGzipMtimeAsync(archivePath, cancellationToken).ConfigureAwait(false); + } + + private static IEnumerable<(string Name, string Path)> EnumerateArchiveEntries(string metadataPath, string dbPath) + { + yield return ("metadata.json", metadataPath); + yield return ("trivy.db", dbPath); + } + + private static async Task ComputeDigestAsync(string archivePath, CancellationToken cancellationToken) + { + await using var stream = new FileStream( + archivePath, + FileMode.Open, + FileAccess.Read, + FileShare.Read, + bufferSize: 81920, + options: FileOptions.Asynchronous | FileOptions.SequentialScan); + var hash = await SHA256.HashDataAsync(stream, cancellationToken).ConfigureAwait(false); + return $"sha256:{Convert.ToHexString(hash).ToLowerInvariant()}"; + } + + private static async Task ZeroGzipMtimeAsync(string archivePath, CancellationToken cancellationToken) + { + await using var stream = new FileStream( + archivePath, + FileMode.Open, + FileAccess.ReadWrite, + FileShare.None, + bufferSize: 8, + options: FileOptions.Asynchronous); + + if (stream.Length < 10) + { + return; + } + + stream.Position = 4; + var zeros = new byte[4]; + await stream.WriteAsync(zeros, cancellationToken).ConfigureAwait(false); + await stream.FlushAsync(cancellationToken).ConfigureAwait(false); + } + + private static string ToGoDuration(TimeSpan span) + { + if (span <= TimeSpan.Zero) + { + return "0s"; + } + + span = span.Duration(); + var builder = new StringBuilder(); + + var totalHours = (int)span.TotalHours; + if (totalHours > 0) + { + builder.Append(totalHours); + builder.Append('h'); + } + + var minutes = span.Minutes; + if (minutes > 0) + { + builder.Append(minutes); + builder.Append('m'); + } + + var seconds = span.Seconds + span.Milliseconds / 1000.0; + if (seconds > 0 || builder.Length == 0) + { + if (span.Milliseconds == 0) + { + builder.Append(span.Seconds); + } + else + { + builder.Append(seconds.ToString("0.###", CultureInfo.InvariantCulture)); + } + builder.Append('s'); + } + + return builder.ToString(); + } + +} diff --git a/src/StellaOps.Feedser.Exporter.TrivyDb/TrivyDbBuilderResult.cs b/src/StellaOps.Feedser.Exporter.TrivyDb/TrivyDbBuilderResult.cs new file mode 100644 index 00000000..cba55dbd --- /dev/null +++ b/src/StellaOps.Feedser.Exporter.TrivyDb/TrivyDbBuilderResult.cs @@ -0,0 +1,10 @@ +using System; + +namespace StellaOps.Feedser.Exporter.TrivyDb; + +public sealed record TrivyDbBuilderResult( + string ArchivePath, + string ArchiveDigest, + long ArchiveLength, + ReadOnlyMemory BuilderMetadata, + string WorkingDirectory); diff --git a/src/StellaOps.Feedser.Exporter.TrivyDb/TrivyDbExportJob.cs b/src/StellaOps.Feedser.Exporter.TrivyDb/TrivyDbExportJob.cs new file mode 100644 index 00000000..3d0acb02 --- /dev/null +++ b/src/StellaOps.Feedser.Exporter.TrivyDb/TrivyDbExportJob.cs @@ -0,0 +1,94 @@ +using System; +using System.Collections.Generic; +using System.Globalization; +using System.Text.Json; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.Extensions.Logging; +using StellaOps.Feedser.Core.Jobs; + +namespace StellaOps.Feedser.Exporter.TrivyDb; + +public sealed class TrivyDbExportJob : IJob +{ + public const string JobKind = "export:trivy-db"; + public static readonly TimeSpan DefaultTimeout = TimeSpan.FromMinutes(20); + public static readonly TimeSpan DefaultLeaseDuration = TimeSpan.FromMinutes(10); + + private readonly TrivyDbFeedExporter _exporter; + private readonly ILogger _logger; + + public TrivyDbExportJob(TrivyDbFeedExporter exporter, ILogger logger) + { + _exporter = exporter ?? throw new ArgumentNullException(nameof(exporter)); + _logger = logger ?? throw new ArgumentNullException(nameof(logger)); + } + + public async Task ExecuteAsync(JobExecutionContext context, CancellationToken cancellationToken) + { + _logger.LogInformation("Executing Trivy DB export job {RunId}", context.RunId); + var overrides = CreateOverrides(context.Parameters); + if (overrides?.HasOverrides == true) + { + using var scope = TrivyDbExportOverrideScope.Begin(overrides); + await _exporter.ExportAsync(context.Services, cancellationToken).ConfigureAwait(false); + } + else + { + await _exporter.ExportAsync(context.Services, cancellationToken).ConfigureAwait(false); + } + + _logger.LogInformation("Completed Trivy DB export job {RunId}", context.RunId); + } + + private static TrivyDbExportOverrides? CreateOverrides(IReadOnlyDictionary parameters) + { + if (parameters is null || parameters.Count == 0) + { + return null; + } + + var publishFull = TryReadBoolean(parameters, "publishFull"); + var publishDelta = TryReadBoolean(parameters, "publishDelta"); + var includeFull = TryReadBoolean(parameters, "includeFull"); + var includeDelta = TryReadBoolean(parameters, "includeDelta"); + + var overrides = new TrivyDbExportOverrides(publishFull, publishDelta, includeFull, includeDelta); + return overrides.HasOverrides ? overrides : null; + } + + private static bool? TryReadBoolean(IReadOnlyDictionary parameters, string key) + { + if (!parameters.TryGetValue(key, out var value) || value is null) + { + return null; + } + + switch (value) + { + case bool b: + return b; + case string s when bool.TryParse(s, out var result): + return result; + case JsonElement element: + return element.ValueKind switch + { + JsonValueKind.True => true, + JsonValueKind.False => false, + JsonValueKind.String when bool.TryParse(element.GetString(), out var parsed) => parsed, + _ => null, + }; + case IConvertible convertible: + try + { + return convertible.ToBoolean(CultureInfo.InvariantCulture); + } + catch + { + return null; + } + } + + return null; + } +} diff --git a/src/StellaOps.Feedser.Exporter.TrivyDb/TrivyDbExportMode.cs b/src/StellaOps.Feedser.Exporter.TrivyDb/TrivyDbExportMode.cs new file mode 100644 index 00000000..25dc5a8c --- /dev/null +++ b/src/StellaOps.Feedser.Exporter.TrivyDb/TrivyDbExportMode.cs @@ -0,0 +1,8 @@ +namespace StellaOps.Feedser.Exporter.TrivyDb; + +public enum TrivyDbExportMode +{ + Full, + Delta, + Skip, +} diff --git a/src/StellaOps.Feedser.Exporter.TrivyDb/TrivyDbExportOptions.cs b/src/StellaOps.Feedser.Exporter.TrivyDb/TrivyDbExportOptions.cs new file mode 100644 index 00000000..8adec7b2 --- /dev/null +++ b/src/StellaOps.Feedser.Exporter.TrivyDb/TrivyDbExportOptions.cs @@ -0,0 +1,88 @@ +using System; +using System.IO; +using System.Collections.Generic; +using StellaOps.Feedser.Exporter.Json; + +namespace StellaOps.Feedser.Exporter.TrivyDb; + +public sealed class TrivyDbExportOptions +{ + public string OutputRoot { get; set; } = Path.Combine("exports", "trivy"); + + public string ReferencePrefix { get; set; } = "feedser/trivy"; + + public string TagFormat { get; set; } = "yyyyMMdd'T'HHmmss'Z'"; + + public string DatabaseVersionFormat { get; set; } = "yyyyMMdd'T'HHmmss'Z'"; + + public bool KeepWorkingTree { get; set; } + + public string? TargetRepository { get; set; } + + public JsonExportOptions Json { get; set; } = new() + { + OutputRoot = Path.Combine("exports", "trivy", "tree") + }; + + public TrivyDbBuilderOptions Builder { get; set; } = new(); + + public TrivyDbOrasOptions Oras { get; set; } = new(); + + public TrivyDbOfflineBundleOptions OfflineBundle { get; set; } = new(); + + public string GetExportRoot(string exportId) + { + ArgumentException.ThrowIfNullOrEmpty(exportId); + var root = Path.GetFullPath(OutputRoot); + return Path.Combine(root, exportId); + } +} + +public sealed class TrivyDbBuilderOptions +{ + public string ExecutablePath { get; set; } = "trivy-db"; + + public string? WorkingDirectory { get; set; } + + public TimeSpan UpdateInterval { get; set; } = TimeSpan.FromHours(24); + + public List OnlyUpdateTargets { get; set; } = new(); + + public Dictionary Environment { get; set; } = new(StringComparer.OrdinalIgnoreCase); + + public bool InheritEnvironment { get; set; } = true; +} + +public sealed class TrivyDbOrasOptions +{ + public bool Enabled { get; set; } + + public string ExecutablePath { get; set; } = "oras"; + + public bool PublishFull { get; set; } = true; + + public bool PublishDelta { get; set; } = true; + + public string? WorkingDirectory { get; set; } + + public bool InheritEnvironment { get; set; } = true; + + public List AdditionalArguments { get; set; } = new(); + + public Dictionary Environment { get; set; } = new(StringComparer.OrdinalIgnoreCase); + + public bool SkipTlsVerify { get; set; } + + public bool UseHttp { get; set; } +} + +public sealed class TrivyDbOfflineBundleOptions +{ + public bool Enabled { get; set; } + + public bool IncludeFull { get; set; } = true; + + public bool IncludeDelta { get; set; } = true; + + public string? FileName { get; set; } +} diff --git a/src/StellaOps.Feedser.Exporter.TrivyDb/TrivyDbExportOverrides.cs b/src/StellaOps.Feedser.Exporter.TrivyDb/TrivyDbExportOverrides.cs new file mode 100644 index 00000000..6da26b0a --- /dev/null +++ b/src/StellaOps.Feedser.Exporter.TrivyDb/TrivyDbExportOverrides.cs @@ -0,0 +1,50 @@ +using System; +using System.Threading; + +namespace StellaOps.Feedser.Exporter.TrivyDb; + +internal sealed record TrivyDbExportOverrides( + bool? PublishFull, + bool? PublishDelta, + bool? IncludeFull, + bool? IncludeDelta) +{ + public bool HasOverrides => + PublishFull.HasValue || PublishDelta.HasValue || IncludeFull.HasValue || IncludeDelta.HasValue; +} + +internal static class TrivyDbExportOverrideScope +{ + private sealed class Scope : IDisposable + { + private readonly TrivyDbExportOverrides? _previous; + private bool _disposed; + + public Scope(TrivyDbExportOverrides? previous) + { + _previous = previous; + } + + public void Dispose() + { + if (_disposed) + { + return; + } + + _disposed = true; + CurrentOverrides.Value = _previous; + } + } + + private static readonly AsyncLocal CurrentOverrides = new(); + + public static TrivyDbExportOverrides? Current => CurrentOverrides.Value; + + public static IDisposable Begin(TrivyDbExportOverrides overrides) + { + var previous = CurrentOverrides.Value; + CurrentOverrides.Value = overrides; + return new Scope(previous); + } +} diff --git a/src/StellaOps.Feedser.Exporter.TrivyDb/TrivyDbExportPlan.cs b/src/StellaOps.Feedser.Exporter.TrivyDb/TrivyDbExportPlan.cs new file mode 100644 index 00000000..f75a2644 --- /dev/null +++ b/src/StellaOps.Feedser.Exporter.TrivyDb/TrivyDbExportPlan.cs @@ -0,0 +1,14 @@ +namespace StellaOps.Feedser.Exporter.TrivyDb; + +using System.Collections.Generic; +using StellaOps.Feedser.Storage.Mongo.Exporting; + +public sealed record TrivyDbExportPlan( + TrivyDbExportMode Mode, + string TreeDigest, + string? BaseExportId, + string? BaseManifestDigest, + bool ResetBaseline, + IReadOnlyList Manifest, + IReadOnlyList ChangedFiles, + IReadOnlyList RemovedPaths); diff --git a/src/StellaOps.Feedser.Exporter.TrivyDb/TrivyDbExportPlanner.cs b/src/StellaOps.Feedser.Exporter.TrivyDb/TrivyDbExportPlanner.cs new file mode 100644 index 00000000..b82b73c7 --- /dev/null +++ b/src/StellaOps.Feedser.Exporter.TrivyDb/TrivyDbExportPlanner.cs @@ -0,0 +1,115 @@ +using System; +using StellaOps.Feedser.Storage.Mongo.Exporting; + +namespace StellaOps.Feedser.Exporter.TrivyDb; + +using System; +using System.Collections.Generic; +using System.Linq; +using StellaOps.Feedser.Storage.Mongo.Exporting; + +public sealed class TrivyDbExportPlanner +{ + public TrivyDbExportPlan CreatePlan( + ExportStateRecord? existingState, + string treeDigest, + IReadOnlyList manifest) + { + ArgumentException.ThrowIfNullOrEmpty(treeDigest); + manifest ??= Array.Empty(); + + if (existingState is null || (existingState.Files?.Count ?? 0) == 0) + { + return new TrivyDbExportPlan( + TrivyDbExportMode.Full, + treeDigest, + BaseExportId: existingState?.BaseExportId, + BaseManifestDigest: existingState?.LastFullDigest, + ResetBaseline: true, + Manifest: manifest, + ChangedFiles: manifest, + RemovedPaths: Array.Empty()); + } + + var existingFiles = existingState.Files ?? Array.Empty(); + var cursorMatches = string.Equals(existingState.ExportCursor, treeDigest, StringComparison.Ordinal); + if (cursorMatches) + { + return new TrivyDbExportPlan( + TrivyDbExportMode.Skip, + treeDigest, + existingState.BaseExportId, + existingState.LastFullDigest, + ResetBaseline: false, + Manifest: existingFiles, + ChangedFiles: Array.Empty(), + RemovedPaths: Array.Empty()); + } + + var existingMap = existingFiles.ToDictionary(static file => file.Path, StringComparer.OrdinalIgnoreCase); + var newMap = manifest.ToDictionary(static file => file.Path, StringComparer.OrdinalIgnoreCase); + + var removed = existingMap.Keys + .Where(path => !newMap.ContainsKey(path)) + .ToArray(); + + if (removed.Length > 0) + { + return new TrivyDbExportPlan( + TrivyDbExportMode.Full, + treeDigest, + existingState.BaseExportId, + existingState.LastFullDigest, + ResetBaseline: true, + Manifest: manifest, + ChangedFiles: manifest, + RemovedPaths: removed); + } + + var changed = new List(); + foreach (var file in manifest) + { + if (!existingMap.TryGetValue(file.Path, out var previous) || !string.Equals(previous.Digest, file.Digest, StringComparison.Ordinal)) + { + changed.Add(file); + } + } + + if (changed.Count == 0) + { + return new TrivyDbExportPlan( + TrivyDbExportMode.Skip, + treeDigest, + existingState.BaseExportId, + existingState.LastFullDigest, + ResetBaseline: false, + Manifest: existingFiles, + ChangedFiles: Array.Empty(), + RemovedPaths: Array.Empty()); + } + + var hasOutstandingDelta = existingState.LastDeltaDigest is not null; + if (hasOutstandingDelta) + { + return new TrivyDbExportPlan( + TrivyDbExportMode.Full, + treeDigest, + existingState.BaseExportId, + existingState.LastFullDigest, + ResetBaseline: true, + Manifest: manifest, + ChangedFiles: manifest, + RemovedPaths: Array.Empty()); + } + + return new TrivyDbExportPlan( + TrivyDbExportMode.Delta, + treeDigest, + existingState.BaseExportId, + existingState.LastFullDigest, + ResetBaseline: false, + Manifest: manifest, + ChangedFiles: changed, + RemovedPaths: Array.Empty()); + } +} diff --git a/src/StellaOps.Feedser.Exporter.TrivyDb/TrivyDbExporterDependencyInjectionRoutine.cs b/src/StellaOps.Feedser.Exporter.TrivyDb/TrivyDbExporterDependencyInjectionRoutine.cs new file mode 100644 index 00000000..c55cb223 --- /dev/null +++ b/src/StellaOps.Feedser.Exporter.TrivyDb/TrivyDbExporterDependencyInjectionRoutine.cs @@ -0,0 +1,64 @@ +using System; +using System.IO; +using Microsoft.Extensions.Configuration; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.DependencyInjection.Extensions; +using Microsoft.Extensions.Options; +using StellaOps.DependencyInjection; +using StellaOps.Feedser.Core.Jobs; +using StellaOps.Feedser.Exporter.Json; +using StellaOps.Feedser.Storage.Mongo.Exporting; + +namespace StellaOps.Feedser.Exporter.TrivyDb; + +public sealed class TrivyDbExporterDependencyInjectionRoutine : IDependencyInjectionRoutine +{ + private const string ConfigurationSection = "feedser:exporters:trivyDb"; + + public IServiceCollection Register(IServiceCollection services, IConfiguration configuration) + { + ArgumentNullException.ThrowIfNull(services); + ArgumentNullException.ThrowIfNull(configuration); + + services.TryAddSingleton(); + services.TryAddSingleton(); + + services.AddOptions() + .Bind(configuration.GetSection(ConfigurationSection)) + .PostConfigure(static options => + { + options.OutputRoot = Normalize(options.OutputRoot, Path.Combine("exports", "trivy")); + options.Json.OutputRoot = Normalize(options.Json.OutputRoot, Path.Combine("exports", "trivy", "tree")); + options.TagFormat = string.IsNullOrWhiteSpace(options.TagFormat) ? "yyyyMMdd'T'HHmmss'Z'" : options.TagFormat; + options.DatabaseVersionFormat = string.IsNullOrWhiteSpace(options.DatabaseVersionFormat) ? "yyyyMMdd'T'HHmmss'Z'" : options.DatabaseVersionFormat; + options.ReferencePrefix = string.IsNullOrWhiteSpace(options.ReferencePrefix) ? "feedser/trivy" : options.ReferencePrefix; + }); + + services.AddSingleton(); + services.AddSingleton(); + services.AddSingleton(); + services.AddSingleton(); + services.AddSingleton(); + services.AddSingleton(); + services.AddTransient(); + + services.PostConfigure(options => + { + if (!options.Definitions.ContainsKey(TrivyDbExportJob.JobKind)) + { + options.Definitions[TrivyDbExportJob.JobKind] = new JobDefinition( + TrivyDbExportJob.JobKind, + typeof(TrivyDbExportJob), + TrivyDbExportJob.DefaultTimeout, + TrivyDbExportJob.DefaultLeaseDuration, + null, + true); + } + }); + + return services; + } + + private static string Normalize(string? value, string fallback) + => string.IsNullOrWhiteSpace(value) ? fallback : value; +} diff --git a/src/StellaOps.Feedser.Exporter.TrivyDb/TrivyDbExporterPlugin.cs b/src/StellaOps.Feedser.Exporter.TrivyDb/TrivyDbExporterPlugin.cs new file mode 100644 index 00000000..aab67973 --- /dev/null +++ b/src/StellaOps.Feedser.Exporter.TrivyDb/TrivyDbExporterPlugin.cs @@ -0,0 +1,23 @@ +using System; +using Microsoft.Extensions.DependencyInjection; +using StellaOps.Feedser.Storage.Mongo.Advisories; +using StellaOps.Plugin; + +namespace StellaOps.Feedser.Exporter.TrivyDb; + +public sealed class TrivyDbExporterPlugin : IExporterPlugin +{ + public string Name => TrivyDbFeedExporter.ExporterName; + + public bool IsAvailable(IServiceProvider services) + { + ArgumentNullException.ThrowIfNull(services); + return services.GetService() is not null; + } + + public IFeedExporter Create(IServiceProvider services) + { + ArgumentNullException.ThrowIfNull(services); + return ActivatorUtilities.CreateInstance(services); + } +} diff --git a/src/StellaOps.Feedser.Exporter.TrivyDb/TrivyDbFeedExporter.cs b/src/StellaOps.Feedser.Exporter.TrivyDb/TrivyDbFeedExporter.cs new file mode 100644 index 00000000..11a1aa88 --- /dev/null +++ b/src/StellaOps.Feedser.Exporter.TrivyDb/TrivyDbFeedExporter.cs @@ -0,0 +1,490 @@ +using System; +using System.Collections.Generic; +using System.Globalization; +using System.IO; +using System.IO.Compression; +using System.Linq; +using System.Security.Cryptography; +using System.Text.Json; +using System.Text.Json.Serialization; +using System.Threading; +using System.Threading.Tasks; +using System.Formats.Tar; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Options; +using StellaOps.Feedser.Exporter.Json; +using StellaOps.Feedser.Models; +using StellaOps.Feedser.Storage.Mongo.Advisories; +using StellaOps.Feedser.Storage.Mongo.Exporting; +using StellaOps.Plugin; + +namespace StellaOps.Feedser.Exporter.TrivyDb; + +public sealed class TrivyDbFeedExporter : IFeedExporter +{ + public const string ExporterName = "trivy-db"; + public const string ExporterId = "export:trivy-db"; + + private readonly IAdvisoryStore _advisoryStore; + private readonly IJsonExportPathResolver _pathResolver; + private readonly TrivyDbExportOptions _options; + private readonly TrivyDbPackageBuilder _packageBuilder; + private readonly TrivyDbOciWriter _ociWriter; + private readonly ExportStateManager _stateManager; + private readonly TrivyDbExportPlanner _exportPlanner; + private readonly ITrivyDbBuilder _builder; + private readonly ITrivyDbOrasPusher _orasPusher; + private readonly ILogger _logger; + private readonly TimeProvider _timeProvider; + private readonly string _exporterVersion; + + public TrivyDbFeedExporter( + IAdvisoryStore advisoryStore, + IJsonExportPathResolver pathResolver, + IOptions options, + TrivyDbPackageBuilder packageBuilder, + TrivyDbOciWriter ociWriter, + ExportStateManager stateManager, + TrivyDbExportPlanner exportPlanner, + ITrivyDbBuilder builder, + ITrivyDbOrasPusher orasPusher, + ILogger logger, + TimeProvider? timeProvider = null) + { + _advisoryStore = advisoryStore ?? throw new ArgumentNullException(nameof(advisoryStore)); + _pathResolver = pathResolver ?? throw new ArgumentNullException(nameof(pathResolver)); + _options = options?.Value ?? throw new ArgumentNullException(nameof(options)); + _packageBuilder = packageBuilder ?? throw new ArgumentNullException(nameof(packageBuilder)); + _ociWriter = ociWriter ?? throw new ArgumentNullException(nameof(ociWriter)); + _stateManager = stateManager ?? throw new ArgumentNullException(nameof(stateManager)); + _exportPlanner = exportPlanner ?? throw new ArgumentNullException(nameof(exportPlanner)); + _builder = builder ?? throw new ArgumentNullException(nameof(builder)); + _orasPusher = orasPusher ?? throw new ArgumentNullException(nameof(orasPusher)); + _logger = logger ?? throw new ArgumentNullException(nameof(logger)); + _timeProvider = timeProvider ?? TimeProvider.System; + _exporterVersion = ExporterVersion.GetVersion(typeof(TrivyDbFeedExporter)); + } + + public string Name => ExporterName; + + public async Task ExportAsync(IServiceProvider services, CancellationToken cancellationToken) + { + var exportedAt = _timeProvider.GetUtcNow(); + var exportId = exportedAt.ToString(_options.TagFormat, CultureInfo.InvariantCulture); + var reference = $"{_options.ReferencePrefix}:{exportId}"; + + _logger.LogInformation("Starting Trivy DB export {ExportId}", exportId); + + var jsonBuilder = new JsonExportSnapshotBuilder(_options.Json, _pathResolver); + var advisories = await LoadAdvisoriesAsync(cancellationToken).ConfigureAwait(false); + var jsonResult = await jsonBuilder.WriteAsync(advisories, exportedAt, exportId, cancellationToken).ConfigureAwait(false); + + _logger.LogInformation( + "Prepared Trivy JSON tree {ExportId} with {AdvisoryCount} advisories ({Bytes} bytes)", + exportId, + jsonResult.AdvisoryCount, + jsonResult.TotalBytes); + + var manifest = jsonResult.Files + .Select(static file => new ExportFileRecord(file.RelativePath, file.Length, file.Digest)) + .ToArray(); + + var treeDigest = ExportDigestCalculator.ComputeTreeDigest(jsonResult); + var existingState = await _stateManager.GetAsync(ExporterId, cancellationToken).ConfigureAwait(false); + var plan = _exportPlanner.CreatePlan(existingState, treeDigest, manifest); + + if (plan.Mode == TrivyDbExportMode.Skip) + { + _logger.LogInformation( + "Trivy DB export {ExportId} unchanged from base {BaseExport}; skipping OCI packaging.", + exportId, + plan.BaseExportId ?? "(none)"); + + if (!_options.KeepWorkingTree) + { + TryDeleteDirectory(jsonResult.ExportDirectory); + } + + return; + } + + if (plan.Mode == TrivyDbExportMode.Delta) + { + _logger.LogInformation( + "Trivy DB export {ExportId} identified {ChangedCount} changed JSON files.", + exportId, + plan.ChangedFiles.Count); + } + + var builderResult = await _builder.BuildAsync(jsonResult, exportedAt, exportId, cancellationToken).ConfigureAwait(false); + var metadataBytes = CreateMetadataJson(plan, builderResult.BuilderMetadata, treeDigest, jsonResult, exportedAt); + + try + { + var package = _packageBuilder.BuildPackage(new TrivyDbPackageRequest( + metadataBytes, + builderResult.ArchivePath, + builderResult.ArchiveDigest, + builderResult.ArchiveLength, + exportedAt, + exportedAt.ToString(_options.DatabaseVersionFormat, CultureInfo.InvariantCulture))); + + var destination = _options.GetExportRoot(exportId); + string? baseLayout = null; + if (plan.Mode == TrivyDbExportMode.Delta && !string.IsNullOrWhiteSpace(plan.BaseExportId)) + { + baseLayout = _options.GetExportRoot(plan.BaseExportId); + } + + var ociResult = await _ociWriter.WriteAsync(package, destination, reference, plan, baseLayout, cancellationToken).ConfigureAwait(false); + + if (_options.Oras.Enabled && ShouldPublishToOras(plan.Mode)) + { + await _orasPusher.PushAsync(destination, reference, exportId, cancellationToken).ConfigureAwait(false); + } + + _logger.LogInformation( + "Trivy DB export {ExportId} wrote manifest {ManifestDigest}", + exportId, + ociResult.ManifestDigest); + + var resetBaseline = plan.ResetBaseline + || existingState is null + || string.IsNullOrWhiteSpace(existingState.BaseExportId) + || string.IsNullOrWhiteSpace(existingState.BaseDigest); + + if (existingState is not null + && !string.IsNullOrWhiteSpace(_options.TargetRepository) + && !string.Equals(existingState.TargetRepository, _options.TargetRepository, StringComparison.Ordinal)) + { + resetBaseline = true; + } + + if (plan.Mode == TrivyDbExportMode.Full || resetBaseline) + { + await _stateManager.StoreFullExportAsync( + ExporterId, + exportId, + ociResult.ManifestDigest, + cursor: treeDigest, + targetRepository: _options.TargetRepository, + exporterVersion: _exporterVersion, + resetBaseline: resetBaseline, + manifest: plan.Manifest, + cancellationToken: cancellationToken).ConfigureAwait(false); + } + else + { + await _stateManager.StoreDeltaExportAsync( + ExporterId, + deltaDigest: treeDigest, + cursor: treeDigest, + exporterVersion: _exporterVersion, + manifest: plan.Manifest, + cancellationToken: cancellationToken).ConfigureAwait(false); + } + + await CreateOfflineBundleAsync(destination, exportId, exportedAt, plan.Mode, cancellationToken).ConfigureAwait(false); + } + finally + { + TryDeleteDirectory(builderResult.WorkingDirectory); + } + + if (!_options.KeepWorkingTree) + { + TryDeleteDirectory(jsonResult.ExportDirectory); + } + } + + private async Task> LoadAdvisoriesAsync(CancellationToken cancellationToken) + { + var advisories = new List(); + await foreach (var advisory in _advisoryStore.StreamAsync(cancellationToken).ConfigureAwait(false)) + { + if (advisory is null) + { + continue; + } + + advisories.Add(advisory); + } + + advisories.Sort(static (left, right) => string.CompareOrdinal(left.AdvisoryKey, right.AdvisoryKey)); + return advisories; + } + + private byte[] CreateMetadataJson( + TrivyDbExportPlan plan, + ReadOnlyMemory builderMetadata, + string treeDigest, + JsonExportResult result, + DateTimeOffset exportedAt) + { + var metadata = new TrivyMetadata + { + GeneratedAt = exportedAt.UtcDateTime, + AdvisoryCount = result.AdvisoryCount, + TreeDigest = treeDigest, + TreeBytes = result.TotalBytes, + ExporterVersion = _exporterVersion, + Builder = ParseBuilderMetadata(builderMetadata.Span), + Mode = plan.Mode switch + { + TrivyDbExportMode.Full => "full", + TrivyDbExportMode.Delta => "delta", + TrivyDbExportMode.Skip => "skip", + _ => "unknown", + }, + BaseExportId = plan.BaseExportId, + BaseManifestDigest = plan.BaseManifestDigest, + ResetBaseline = plan.ResetBaseline, + Delta = plan.Mode == TrivyDbExportMode.Delta + ? new DeltaMetadata( + plan.ChangedFiles.Select(static file => new DeltaFileMetadata(file.Path, file.Length, file.Digest)).ToArray(), + plan.RemovedPaths.ToArray()) + : null, + }; + + return JsonSerializer.SerializeToUtf8Bytes(metadata, new JsonSerializerOptions + { + PropertyNamingPolicy = JsonNamingPolicy.CamelCase, + DefaultIgnoreCondition = JsonIgnoreCondition.WhenWritingNull, + WriteIndented = false, + }); + } + + private static BuilderMetadata? ParseBuilderMetadata(ReadOnlySpan payload) + { + if (payload.IsEmpty) + { + return null; + } + + try + { + return JsonSerializer.Deserialize(payload, new JsonSerializerOptions + { + PropertyNameCaseInsensitive = true, + }); + } + catch + { + return null; + } + } + + private async Task CreateOfflineBundleAsync(string layoutPath, string exportId, DateTimeOffset exportedAt, TrivyDbExportMode mode, CancellationToken cancellationToken) + { + if (!_options.OfflineBundle.Enabled) + { + return; + } + + if (!ShouldIncludeInBundle(mode)) + { + return; + } + + var parent = Path.GetDirectoryName(layoutPath) ?? layoutPath; + var fileName = string.IsNullOrWhiteSpace(_options.OfflineBundle.FileName) + ? $"{exportId}.offline.tar.gz" + : _options.OfflineBundle.FileName.Replace("{exportId}", exportId, StringComparison.Ordinal); + + var bundlePath = Path.IsPathRooted(fileName) ? fileName : Path.Combine(parent, fileName); + Directory.CreateDirectory(Path.GetDirectoryName(bundlePath)!); + + if (File.Exists(bundlePath)) + { + File.Delete(bundlePath); + } + + var normalizedRoot = Path.GetFullPath(layoutPath); + var directories = Directory.GetDirectories(normalizedRoot, "*", SearchOption.AllDirectories) + .Select(dir => NormalizeTarPath(normalizedRoot, dir) + "/") + .OrderBy(static path => path, StringComparer.Ordinal) + .ToArray(); + + var files = Directory.GetFiles(normalizedRoot, "*", SearchOption.AllDirectories) + .Select(file => NormalizeTarPath(normalizedRoot, file)) + .OrderBy(static path => path, StringComparer.Ordinal) + .ToArray(); + + await using (var archiveStream = new FileStream( + bundlePath, + FileMode.Create, + FileAccess.Write, + FileShare.None, + bufferSize: 81920, + options: FileOptions.Asynchronous | FileOptions.SequentialScan)) + await using (var gzip = new GZipStream(archiveStream, CompressionLevel.SmallestSize, leaveOpen: true)) + await using (var writer = new TarWriter(gzip, TarEntryFormat.Pax, leaveOpen: false)) + { + var timestamp = exportedAt.UtcDateTime; + + foreach (var directory in directories) + { + var entry = new PaxTarEntry(TarEntryType.Directory, directory) + { + ModificationTime = timestamp, + Mode = UnixFileMode.UserRead | UnixFileMode.UserWrite | UnixFileMode.UserExecute | + UnixFileMode.GroupRead | UnixFileMode.GroupExecute | + UnixFileMode.OtherRead | UnixFileMode.OtherExecute, + }; + + writer.WriteEntry(entry); + } + + foreach (var relativePath in files) + { + var fullPath = Path.Combine(normalizedRoot, relativePath.Replace('/', Path.DirectorySeparatorChar)); + var entry = new PaxTarEntry(TarEntryType.RegularFile, relativePath) + { + ModificationTime = timestamp, + Mode = UnixFileMode.UserRead | UnixFileMode.UserWrite | + UnixFileMode.GroupRead | + UnixFileMode.OtherRead, + }; + + await using var source = new FileStream( + fullPath, + FileMode.Open, + FileAccess.Read, + FileShare.Read, + bufferSize: 81920, + options: FileOptions.Asynchronous | FileOptions.SequentialScan); + entry.DataStream = source; + writer.WriteEntry(entry); + } + } + + await ZeroGzipMtimeAsync(bundlePath, cancellationToken).ConfigureAwait(false); + + var digest = await ComputeSha256Async(bundlePath, cancellationToken).ConfigureAwait(false); + var length = new FileInfo(bundlePath).Length; + _logger.LogInformation("Wrote offline bundle {BundlePath} ({Length} bytes, digest {Digest})", bundlePath, length, digest); + } + + private static void TryDeleteDirectory(string directory) + { + try + { + if (Directory.Exists(directory)) + { + Directory.Delete(directory, recursive: true); + } + } + catch + { + // Best effort cleanup – ignore failures. + } + } + + private static async Task ZeroGzipMtimeAsync(string archivePath, CancellationToken cancellationToken) + { + await using var stream = new FileStream( + archivePath, + FileMode.Open, + FileAccess.ReadWrite, + FileShare.None, + bufferSize: 8, + options: FileOptions.Asynchronous); + + if (stream.Length < 10) + { + return; + } + + stream.Position = 4; + var zeros = new byte[4]; + await stream.WriteAsync(zeros, cancellationToken).ConfigureAwait(false); + await stream.FlushAsync(cancellationToken).ConfigureAwait(false); + } + + private static async Task ComputeSha256Async(string path, CancellationToken cancellationToken) + { + await using var stream = new FileStream( + path, + FileMode.Open, + FileAccess.Read, + FileShare.Read, + bufferSize: 81920, + options: FileOptions.Asynchronous | FileOptions.SequentialScan); + var hash = await SHA256.HashDataAsync(stream, cancellationToken).ConfigureAwait(false); + return $"sha256:{Convert.ToHexString(hash).ToLowerInvariant()}"; + } + + private static string NormalizeTarPath(string root, string fullPath) + { + var relative = Path.GetRelativePath(root, fullPath); + var normalized = relative.Replace(Path.DirectorySeparatorChar, '/'); + return string.IsNullOrEmpty(normalized) ? "." : normalized; + } + + private bool ShouldPublishToOras(TrivyDbExportMode mode) + { + var overrides = TrivyDbExportOverrideScope.Current; + + return mode switch + { + TrivyDbExportMode.Full => overrides?.PublishFull ?? _options.Oras.PublishFull, + TrivyDbExportMode.Delta => overrides?.PublishDelta ?? _options.Oras.PublishDelta, + _ => false, + }; + } + + private bool ShouldIncludeInBundle(TrivyDbExportMode mode) + { + var overrides = TrivyDbExportOverrideScope.Current; + + return mode switch + { + TrivyDbExportMode.Full => overrides?.IncludeFull ?? _options.OfflineBundle.IncludeFull, + TrivyDbExportMode.Delta => overrides?.IncludeDelta ?? _options.OfflineBundle.IncludeDelta, + _ => false, + }; + } + + private sealed record DeltaMetadata( + IReadOnlyList ChangedFiles, + IReadOnlyList RemovedPaths); + + private sealed record DeltaFileMetadata(string Path, long Length, string Digest); + + private sealed class TrivyMetadata + { + public DateTime GeneratedAt { get; set; } + + public int AdvisoryCount { get; set; } + + public string TreeDigest { get; set; } = string.Empty; + + public long TreeBytes { get; set; } + + public string ExporterVersion { get; set; } = string.Empty; + + public BuilderMetadata? Builder { get; set; } + + public string Mode { get; set; } = "full"; + + public string? BaseExportId { get; set; } + + public string? BaseManifestDigest { get; set; } + + public bool ResetBaseline { get; set; } + + public DeltaMetadata? Delta { get; set; } + } + + private sealed class BuilderMetadata + { + [JsonPropertyName("Version")] + public int Version { get; set; } + + public DateTime NextUpdate { get; set; } + + public DateTime UpdatedAt { get; set; } + + public DateTime? DownloadedAt { get; set; } + } +} diff --git a/src/StellaOps.Feedser.Exporter.TrivyDb/TrivyDbMediaTypes.cs b/src/StellaOps.Feedser.Exporter.TrivyDb/TrivyDbMediaTypes.cs new file mode 100644 index 00000000..cf667440 --- /dev/null +++ b/src/StellaOps.Feedser.Exporter.TrivyDb/TrivyDbMediaTypes.cs @@ -0,0 +1,9 @@ +namespace StellaOps.Feedser.Exporter.TrivyDb; + +public static class TrivyDbMediaTypes +{ + public const string OciManifest = "application/vnd.oci.image.manifest.v1+json"; + public const string OciImageIndex = "application/vnd.oci.image.index.v1+json"; + public const string TrivyConfig = "application/vnd.aquasec.trivy.config.v1+json"; + public const string TrivyLayer = "application/vnd.aquasec.trivy.db.layer.v1.tar+gzip"; +} diff --git a/src/StellaOps.Feedser.Exporter.TrivyDb/TrivyDbOciWriteResult.cs b/src/StellaOps.Feedser.Exporter.TrivyDb/TrivyDbOciWriteResult.cs new file mode 100644 index 00000000..3a22a7ab --- /dev/null +++ b/src/StellaOps.Feedser.Exporter.TrivyDb/TrivyDbOciWriteResult.cs @@ -0,0 +1,8 @@ +using System.Collections.Generic; + +namespace StellaOps.Feedser.Exporter.TrivyDb; + +public sealed record TrivyDbOciWriteResult( + string RootDirectory, + string ManifestDigest, + IReadOnlyCollection BlobDigests); diff --git a/src/StellaOps.Feedser.Exporter.TrivyDb/TrivyDbOciWriter.cs b/src/StellaOps.Feedser.Exporter.TrivyDb/TrivyDbOciWriter.cs new file mode 100644 index 00000000..eaf2a74d --- /dev/null +++ b/src/StellaOps.Feedser.Exporter.TrivyDb/TrivyDbOciWriter.cs @@ -0,0 +1,375 @@ +using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using System.Text; +using System.Text.Json; +using System.Threading; +using System.Threading.Tasks; + +namespace StellaOps.Feedser.Exporter.TrivyDb; + +/// +/// Writes a Trivy DB package to an OCI image layout directory with deterministic content. +/// +public sealed class TrivyDbOciWriter +{ + private static readonly JsonSerializerOptions SerializerOptions = new() + { + PropertyNamingPolicy = JsonNamingPolicy.CamelCase, + DefaultIgnoreCondition = System.Text.Json.Serialization.JsonIgnoreCondition.WhenWritingNull, + WriteIndented = false, + }; + + private static readonly byte[] OciLayoutBytes = Encoding.UTF8.GetBytes("{\"imageLayoutVersion\":\"1.0.0\"}"); + + public async Task WriteAsync( + TrivyDbPackage package, + string destination, + string reference, + TrivyDbExportPlan plan, + string? baseLayoutPath, + CancellationToken cancellationToken) + { + if (package is null) + { + throw new ArgumentNullException(nameof(package)); + } + + if (string.IsNullOrWhiteSpace(destination)) + { + throw new ArgumentException("Destination directory must be provided.", nameof(destination)); + } + + if (string.IsNullOrWhiteSpace(reference)) + { + throw new ArgumentException("Reference tag must be provided.", nameof(reference)); + } + + if (plan is null) + { + throw new ArgumentNullException(nameof(plan)); + } + + var root = Path.GetFullPath(destination); + if (Directory.Exists(root)) + { + Directory.Delete(root, recursive: true); + } + + Directory.CreateDirectory(root); + var timestamp = package.Config.GeneratedAt.UtcDateTime; + + await WriteFileAsync(Path.Combine(root, "metadata.json"), package.MetadataJson, timestamp, cancellationToken).ConfigureAwait(false); + await WriteFileAsync(Path.Combine(root, "oci-layout"), OciLayoutBytes, timestamp, cancellationToken).ConfigureAwait(false); + + var blobsRoot = Path.Combine(root, "blobs", "sha256"); + Directory.CreateDirectory(blobsRoot); + Directory.SetLastWriteTimeUtc(Path.GetDirectoryName(blobsRoot)!, timestamp); + Directory.SetLastWriteTimeUtc(blobsRoot, timestamp); + + var writtenDigests = new HashSet(StringComparer.Ordinal); + foreach (var pair in package.Blobs) + { + if (writtenDigests.Contains(pair.Key)) + { + continue; + } + + var reused = await TryReuseExistingBlobAsync(baseLayoutPath, pair.Key, blobsRoot, timestamp, cancellationToken).ConfigureAwait(false); + if (reused) + { + writtenDigests.Add(pair.Key); + continue; + } + + if (writtenDigests.Add(pair.Key)) + { + await WriteBlobAsync(blobsRoot, pair.Key, pair.Value, timestamp, cancellationToken).ConfigureAwait(false); + } + } + + var manifestBytes = JsonSerializer.SerializeToUtf8Bytes(package.Manifest, SerializerOptions); + var manifestDigest = ComputeDigest(manifestBytes); + if (!writtenDigests.Contains(manifestDigest)) + { + var reused = await TryReuseExistingBlobAsync(baseLayoutPath, manifestDigest, blobsRoot, timestamp, cancellationToken).ConfigureAwait(false); + if (!reused) + { + await WriteBlobAsync(blobsRoot, manifestDigest, TrivyDbBlob.FromBytes(manifestBytes), timestamp, cancellationToken).ConfigureAwait(false); + } + + writtenDigests.Add(manifestDigest); + } + + var manifestDescriptor = new OciDescriptor( + TrivyDbMediaTypes.OciManifest, + manifestDigest, + manifestBytes.LongLength, + new Dictionary + { + ["org.opencontainers.image.ref.name"] = reference, + }); + var index = new OciIndex(2, new[] { manifestDescriptor }); + var indexBytes = JsonSerializer.SerializeToUtf8Bytes(index, SerializerOptions); + await WriteFileAsync(Path.Combine(root, "index.json"), indexBytes, timestamp, cancellationToken).ConfigureAwait(false); + + if (plan.Mode == TrivyDbExportMode.Delta && !string.IsNullOrWhiteSpace(baseLayoutPath)) + { + var reuseDigests = await TryReuseBaseBlobsAsync( + blobsRoot, + timestamp, + writtenDigests, + plan, + baseLayoutPath, + cancellationToken).ConfigureAwait(false); + foreach (var digest in reuseDigests) + { + writtenDigests.Add(digest); + } + } + + Directory.SetLastWriteTimeUtc(root, timestamp); + + var blobDigests = writtenDigests.ToArray(); + Array.Sort(blobDigests, StringComparer.Ordinal); + return new TrivyDbOciWriteResult(root, manifestDigest, blobDigests); + } + + private static async Task WriteFileAsync(string path, ReadOnlyMemory bytes, DateTime utcTimestamp, CancellationToken cancellationToken) + { + var directory = Path.GetDirectoryName(path); + if (!string.IsNullOrEmpty(directory)) + { + Directory.CreateDirectory(directory); + Directory.SetLastWriteTimeUtc(directory, utcTimestamp); + } + + await using var destination = new FileStream( + path, + FileMode.Create, + FileAccess.Write, + FileShare.None, + bufferSize: 81920, + options: FileOptions.Asynchronous | FileOptions.SequentialScan); + await destination.WriteAsync(bytes, cancellationToken).ConfigureAwait(false); + await destination.FlushAsync(cancellationToken).ConfigureAwait(false); + File.SetLastWriteTimeUtc(path, utcTimestamp); + } + + private static async Task WriteBlobAsync(string blobsRoot, string digest, TrivyDbBlob blob, DateTime utcTimestamp, CancellationToken cancellationToken) + { + var fileName = ResolveDigestFileName(digest); + var path = Path.Combine(blobsRoot, fileName); + var directory = Path.GetDirectoryName(path); + if (!string.IsNullOrEmpty(directory)) + { + Directory.CreateDirectory(directory); + Directory.SetLastWriteTimeUtc(directory, utcTimestamp); + } + + await using var source = await blob.OpenReadAsync(cancellationToken).ConfigureAwait(false); + await using var destination = new FileStream( + path, + FileMode.Create, + FileAccess.Write, + FileShare.None, + bufferSize: 81920, + options: FileOptions.Asynchronous | FileOptions.SequentialScan); + + await source.CopyToAsync(destination, cancellationToken).ConfigureAwait(false); + await destination.FlushAsync(cancellationToken).ConfigureAwait(false); + File.SetLastWriteTimeUtc(path, utcTimestamp); + } + + private static string ResolveDigestFileName(string digest) + { + if (!digest.StartsWith("sha256:", StringComparison.Ordinal)) + { + throw new InvalidOperationException($"Only sha256 digests are supported. Received '{digest}'."); + } + + var hex = digest[7..]; + if (hex.Length == 0) + { + throw new InvalidOperationException("Digest hex component cannot be empty."); + } + + return hex; + } + + private static string ComputeDigest(ReadOnlySpan payload) + { + var hash = System.Security.Cryptography.SHA256.HashData(payload); + var hex = Convert.ToHexString(hash); + Span buffer = stackalloc char[7 + hex.Length]; // "sha256:" + hex + buffer[0] = 's'; + buffer[1] = 'h'; + buffer[2] = 'a'; + buffer[3] = '2'; + buffer[4] = '5'; + buffer[5] = '6'; + buffer[6] = ':'; + for (var i = 0; i < hex.Length; i++) + { + buffer[7 + i] = char.ToLowerInvariant(hex[i]); + } + + return new string(buffer); + } + + private static async Task> TryReuseBaseBlobsAsync( + string destinationBlobsRoot, + DateTime timestamp, + HashSet written, + TrivyDbExportPlan plan, + string baseLayoutPath, + CancellationToken cancellationToken) + { + if (string.IsNullOrWhiteSpace(plan.BaseManifestDigest)) + { + return Array.Empty(); + } + + var baseRoot = Path.GetFullPath(baseLayoutPath); + if (!Directory.Exists(baseRoot)) + { + return Array.Empty(); + } + + var manifestPath = ResolveBlobPath(baseRoot, plan.BaseManifestDigest); + if (!File.Exists(manifestPath)) + { + return Array.Empty(); + } + + await using var stream = new FileStream( + manifestPath, + FileMode.Open, + FileAccess.Read, + FileShare.Read, + bufferSize: 81920, + options: FileOptions.Asynchronous | FileOptions.SequentialScan); + using var document = await JsonDocument.ParseAsync(stream, cancellationToken: cancellationToken).ConfigureAwait(false); + var root = document.RootElement; + + var digests = new SortedSet(StringComparer.Ordinal) + { + plan.BaseManifestDigest, + }; + + if (root.TryGetProperty("config", out var configNode)) + { + var digest = configNode.GetProperty("digest").GetString(); + if (!string.IsNullOrWhiteSpace(digest)) + { + digests.Add(digest); + } + } + + if (root.TryGetProperty("layers", out var layersNode)) + { + foreach (var layer in layersNode.EnumerateArray()) + { + var digest = layer.GetProperty("digest").GetString(); + if (!string.IsNullOrWhiteSpace(digest)) + { + digests.Add(digest); + } + } + } + + var copied = new List(); + foreach (var digest in digests) + { + if (written.Contains(digest)) + { + continue; + } + + var sourcePath = ResolveBlobPath(baseRoot, digest); + if (!File.Exists(sourcePath)) + { + continue; + } + + var destinationPath = Path.Combine(destinationBlobsRoot, ResolveDigestFileName(digest)); + Directory.CreateDirectory(Path.GetDirectoryName(destinationPath)!); + await using var source = new FileStream( + sourcePath, + FileMode.Open, + FileAccess.Read, + FileShare.Read, + bufferSize: 81920, + options: FileOptions.Asynchronous | FileOptions.SequentialScan); + await using var destination = new FileStream( + destinationPath, + FileMode.Create, + FileAccess.Write, + FileShare.None, + bufferSize: 81920, + options: FileOptions.Asynchronous | FileOptions.SequentialScan); + await source.CopyToAsync(destination, cancellationToken).ConfigureAwait(false); + await destination.FlushAsync(cancellationToken).ConfigureAwait(false); + File.SetLastWriteTimeUtc(destinationPath, timestamp); + copied.Add(digest); + } + + if (copied.Count > 0) + { + Directory.SetLastWriteTimeUtc(destinationBlobsRoot, timestamp); + Directory.SetLastWriteTimeUtc(Path.GetDirectoryName(destinationBlobsRoot)!, timestamp); + } + + return copied; + } + + private static string ResolveBlobPath(string layoutRoot, string digest) + { + var fileName = ResolveDigestFileName(digest); + return Path.Combine(layoutRoot, "blobs", "sha256", fileName); + } + + private static async Task TryReuseExistingBlobAsync( + string? baseLayoutPath, + string digest, + string destinationBlobsRoot, + DateTime timestamp, + CancellationToken cancellationToken) + { + if (string.IsNullOrWhiteSpace(baseLayoutPath)) + { + return false; + } + + var baseRoot = Path.GetFullPath(baseLayoutPath); + var sourcePath = ResolveBlobPath(baseRoot, digest); + if (!File.Exists(sourcePath)) + { + return false; + } + + var destinationPath = Path.Combine(destinationBlobsRoot, ResolveDigestFileName(digest)); + Directory.CreateDirectory(Path.GetDirectoryName(destinationPath)!); + await using var source = new FileStream( + sourcePath, + FileMode.Open, + FileAccess.Read, + FileShare.Read, + bufferSize: 81920, + options: FileOptions.Asynchronous | FileOptions.SequentialScan); + await using var destination = new FileStream( + destinationPath, + FileMode.Create, + FileAccess.Write, + FileShare.None, + bufferSize: 81920, + options: FileOptions.Asynchronous | FileOptions.SequentialScan); + await source.CopyToAsync(destination, cancellationToken).ConfigureAwait(false); + await destination.FlushAsync(cancellationToken).ConfigureAwait(false); + File.SetLastWriteTimeUtc(destinationPath, timestamp); + Directory.SetLastWriteTimeUtc(destinationBlobsRoot, timestamp); + Directory.SetLastWriteTimeUtc(Path.GetDirectoryName(destinationBlobsRoot)!, timestamp); + return true; + } +} diff --git a/src/StellaOps.Feedser.Exporter.TrivyDb/TrivyDbOrasPusher.cs b/src/StellaOps.Feedser.Exporter.TrivyDb/TrivyDbOrasPusher.cs new file mode 100644 index 00000000..5a723a01 --- /dev/null +++ b/src/StellaOps.Feedser.Exporter.TrivyDb/TrivyDbOrasPusher.cs @@ -0,0 +1,209 @@ +using System; +using System.Diagnostics; +using System.IO; +using System.Linq; +using System.Text; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Options; + +namespace StellaOps.Feedser.Exporter.TrivyDb; + +public sealed class TrivyDbOrasPusher : ITrivyDbOrasPusher +{ + private readonly TrivyDbExportOptions _options; + private readonly ILogger _logger; + + public TrivyDbOrasPusher(IOptions options, ILogger logger) + { + _options = options?.Value ?? throw new ArgumentNullException(nameof(options)); + _logger = logger ?? throw new ArgumentNullException(nameof(logger)); + } + + public async Task PushAsync(string layoutPath, string reference, string exportId, CancellationToken cancellationToken) + { + var orasOptions = _options.Oras; + if (!orasOptions.Enabled) + { + return; + } + + if (string.IsNullOrWhiteSpace(reference)) + { + throw new InvalidOperationException("ORAS push requested but reference is empty."); + } + + if (!Directory.Exists(layoutPath)) + { + throw new DirectoryNotFoundException($"OCI layout directory '{layoutPath}' does not exist."); + } + + var executable = string.IsNullOrWhiteSpace(orasOptions.ExecutablePath) ? "oras" : orasOptions.ExecutablePath; + var tag = ResolveTag(reference, exportId); + var layoutReference = $"{layoutPath}:{tag}"; + + var startInfo = new ProcessStartInfo + { + FileName = executable, + RedirectStandardOutput = true, + RedirectStandardError = true, + UseShellExecute = false, + }; + + startInfo.ArgumentList.Add("cp"); + startInfo.ArgumentList.Add("--from-oci-layout"); + startInfo.ArgumentList.Add(layoutReference); + if (orasOptions.SkipTlsVerify) + { + startInfo.ArgumentList.Add("--insecure"); + } + if (orasOptions.UseHttp) + { + startInfo.ArgumentList.Add("--plain-http"); + } + + if (orasOptions.AdditionalArguments is { Count: > 0 }) + { + foreach (var arg in orasOptions.AdditionalArguments) + { + if (!string.IsNullOrWhiteSpace(arg)) + { + startInfo.ArgumentList.Add(arg); + } + } + } + + startInfo.ArgumentList.Add(reference); + + if (!string.IsNullOrWhiteSpace(orasOptions.WorkingDirectory)) + { + startInfo.WorkingDirectory = orasOptions.WorkingDirectory; + } + + if (!orasOptions.InheritEnvironment) + { + startInfo.Environment.Clear(); + } + + if (orasOptions.Environment is { Count: > 0 }) + { + foreach (var kvp in orasOptions.Environment) + { + if (!string.IsNullOrEmpty(kvp.Key)) + { + startInfo.Environment[kvp.Key] = kvp.Value; + } + } + } + + using var process = new Process { StartInfo = startInfo }; + var stdout = new StringBuilder(); + var stderr = new StringBuilder(); + var stdoutCompletion = new TaskCompletionSource(); + var stderrCompletion = new TaskCompletionSource(); + + process.OutputDataReceived += (_, e) => + { + if (e.Data is null) + { + stdoutCompletion.TrySetResult(null); + } + else + { + stdout.AppendLine(e.Data); + } + }; + + process.ErrorDataReceived += (_, e) => + { + if (e.Data is null) + { + stderrCompletion.TrySetResult(null); + } + else + { + stderr.AppendLine(e.Data); + } + }; + + _logger.LogInformation("Pushing Trivy DB export {ExportId} to {Reference} using {Executable}", exportId, reference, executable); + + try + { + if (!process.Start()) + { + throw new InvalidOperationException($"Failed to start '{executable}'."); + } + } + catch (Exception ex) + { + throw new InvalidOperationException($"Failed to start '{executable}'.", ex); + } + + process.BeginOutputReadLine(); + process.BeginErrorReadLine(); + + using var registration = cancellationToken.Register(() => + { + try + { + if (!process.HasExited) + { + process.Kill(entireProcessTree: true); + } + } + catch + { + // ignore + } + }); + +#if NET8_0_OR_GREATER + await process.WaitForExitAsync(cancellationToken).ConfigureAwait(false); +#else + await Task.Run(() => process.WaitForExit(), cancellationToken).ConfigureAwait(false); +#endif + + await Task.WhenAll(stdoutCompletion.Task, stderrCompletion.Task).ConfigureAwait(false); + + if (process.ExitCode != 0) + { + _logger.LogError("ORAS push for {Reference} failed with code {Code}. stderr: {Stderr}", reference, process.ExitCode, stderr.ToString()); + throw new InvalidOperationException($"'{executable}' exited with code {process.ExitCode}."); + } + + if (stdout.Length > 0) + { + _logger.LogDebug("ORAS push output: {Stdout}", stdout.ToString()); + } + + if (stderr.Length > 0) + { + _logger.LogWarning("ORAS push warnings: {Stderr}", stderr.ToString()); + } + } + + private static string ResolveTag(string reference, string fallback) + { + if (string.IsNullOrWhiteSpace(reference)) + { + return fallback; + } + + var atIndex = reference.IndexOf('@'); + if (atIndex >= 0) + { + reference = reference[..atIndex]; + } + + var slashIndex = reference.LastIndexOf('/'); + var colonIndex = reference.LastIndexOf(':'); + if (colonIndex > slashIndex && colonIndex >= 0) + { + return reference[(colonIndex + 1)..]; + } + + return fallback; + } +} diff --git a/src/StellaOps.Feedser.Exporter.TrivyDb/TrivyDbPackage.cs b/src/StellaOps.Feedser.Exporter.TrivyDb/TrivyDbPackage.cs new file mode 100644 index 00000000..c2d842da --- /dev/null +++ b/src/StellaOps.Feedser.Exporter.TrivyDb/TrivyDbPackage.cs @@ -0,0 +1,9 @@ +using System.Collections.Generic; + +namespace StellaOps.Feedser.Exporter.TrivyDb; + +public sealed record TrivyDbPackage( + OciManifest Manifest, + TrivyConfigDocument Config, + IReadOnlyDictionary Blobs, + ReadOnlyMemory MetadataJson); diff --git a/src/StellaOps.Feedser.Exporter.TrivyDb/TrivyDbPackageBuilder.cs b/src/StellaOps.Feedser.Exporter.TrivyDb/TrivyDbPackageBuilder.cs new file mode 100644 index 00000000..2f5c8a45 --- /dev/null +++ b/src/StellaOps.Feedser.Exporter.TrivyDb/TrivyDbPackageBuilder.cs @@ -0,0 +1,116 @@ +using System; +using System.Collections.Generic; +using System.Collections.Immutable; +using System.IO; +using System.Security.Cryptography; +using System.Text.Json; +using System.Text.Json.Serialization; + +namespace StellaOps.Feedser.Exporter.TrivyDb; + +public sealed class TrivyDbPackageBuilder +{ + private static readonly JsonSerializerOptions SerializerOptions = new() + { + PropertyNamingPolicy = JsonNamingPolicy.CamelCase, + DefaultIgnoreCondition = JsonIgnoreCondition.WhenWritingNull, + WriteIndented = false, + }; + + public TrivyDbPackage BuildPackage(TrivyDbPackageRequest request) + { + if (request is null) + { + throw new ArgumentNullException(nameof(request)); + } + + if (request.MetadataJson.IsEmpty) + { + throw new ArgumentException("Metadata JSON payload must be provided.", nameof(request)); + } + + if (string.IsNullOrWhiteSpace(request.DatabaseArchivePath)) + { + throw new ArgumentException("Database archive path must be provided.", nameof(request)); + } + + if (!File.Exists(request.DatabaseArchivePath)) + { + throw new FileNotFoundException("Database archive path not found.", request.DatabaseArchivePath); + } + + if (string.IsNullOrWhiteSpace(request.DatabaseDigest)) + { + throw new ArgumentException("Database archive digest must be provided.", nameof(request)); + } + + if (request.DatabaseLength < 0) + { + throw new ArgumentOutOfRangeException(nameof(request.DatabaseLength)); + } + + var metadataBytes = request.MetadataJson; + var generatedAt = request.GeneratedAt.ToUniversalTime(); + var configDocument = new TrivyConfigDocument( + TrivyDbMediaTypes.TrivyConfig, + generatedAt, + request.DatabaseVersion, + request.DatabaseDigest, + request.DatabaseLength); + + var configBytes = JsonSerializer.SerializeToUtf8Bytes(configDocument, SerializerOptions); + var configDigest = ComputeDigest(configBytes); + + var configDescriptor = new OciDescriptor( + TrivyDbMediaTypes.TrivyConfig, + configDigest, + configBytes.LongLength, + new Dictionary + { + ["org.opencontainers.image.title"] = "config.json", + }); + + var layerDescriptor = new OciDescriptor( + TrivyDbMediaTypes.TrivyLayer, + request.DatabaseDigest, + request.DatabaseLength, + new Dictionary + { + ["org.opencontainers.image.title"] = "db.tar.gz", + }); + + var manifest = new OciManifest( + 2, + TrivyDbMediaTypes.OciManifest, + configDescriptor, + ImmutableArray.Create(layerDescriptor)); + + var blobs = new SortedDictionary(StringComparer.Ordinal) + { + [configDigest] = TrivyDbBlob.FromBytes(configBytes), + [request.DatabaseDigest] = TrivyDbBlob.FromFile(request.DatabaseArchivePath, request.DatabaseLength), + }; + + return new TrivyDbPackage(manifest, configDocument, blobs, metadataBytes); + } + + private static string ComputeDigest(ReadOnlySpan payload) + { + var hash = SHA256.HashData(payload); + var hex = Convert.ToHexString(hash); + Span buffer = stackalloc char[7 + hex.Length]; // "sha256:" + hex + buffer[0] = 's'; + buffer[1] = 'h'; + buffer[2] = 'a'; + buffer[3] = '2'; + buffer[4] = '5'; + buffer[5] = '6'; + buffer[6] = ':'; + for (var i = 0; i < hex.Length; i++) + { + buffer[7 + i] = char.ToLowerInvariant(hex[i]); + } + + return new string(buffer); + } +} diff --git a/src/StellaOps.Feedser.Exporter.TrivyDb/TrivyDbPackageRequest.cs b/src/StellaOps.Feedser.Exporter.TrivyDb/TrivyDbPackageRequest.cs new file mode 100644 index 00000000..e39618a2 --- /dev/null +++ b/src/StellaOps.Feedser.Exporter.TrivyDb/TrivyDbPackageRequest.cs @@ -0,0 +1,11 @@ +using System; + +namespace StellaOps.Feedser.Exporter.TrivyDb; + +public sealed record TrivyDbPackageRequest( + ReadOnlyMemory MetadataJson, + string DatabaseArchivePath, + string DatabaseDigest, + long DatabaseLength, + DateTimeOffset GeneratedAt, + string DatabaseVersion); diff --git a/src/StellaOps.Feedser.Merge.Tests/AdvisoryIdentityResolverTests.cs b/src/StellaOps.Feedser.Merge.Tests/AdvisoryIdentityResolverTests.cs new file mode 100644 index 00000000..8924a18a --- /dev/null +++ b/src/StellaOps.Feedser.Merge.Tests/AdvisoryIdentityResolverTests.cs @@ -0,0 +1,92 @@ +using System; +using System.Linq; +using StellaOps.Feedser.Merge.Identity; +using StellaOps.Feedser.Models; +using Xunit; + +namespace StellaOps.Feedser.Merge.Tests; + +public sealed class AdvisoryIdentityResolverTests +{ + private readonly AdvisoryIdentityResolver _resolver = new(); + + [Fact] + public void Resolve_GroupsBySharedCveAlias() + { + var nvd = CreateAdvisory("CVE-2025-1234", aliases: new[] { "CVE-2025-1234" }, source: "nvd"); + var vendor = CreateAdvisory("VSA-2025-01", aliases: new[] { "CVE-2025-1234", "VSA-2025-01" }, source: "vendor"); + + var clusters = _resolver.Resolve(new[] { nvd, vendor }); + + var cluster = Assert.Single(clusters); + Assert.Equal("CVE-2025-1234", cluster.AdvisoryKey); + Assert.Equal(2, cluster.Advisories.Length); + Assert.True(cluster.Aliases.Any(alias => alias.Value == "CVE-2025-1234")); + } + + [Fact] + public void Resolve_PrefersPsirtAliasWhenNoCve() + { + var vendor = CreateAdvisory("VMSA-2025-0001", aliases: new[] { "VMSA-2025-0001" }, source: "vmware"); + var osv = CreateAdvisory("OSV-2025-1", aliases: new[] { "OSV-2025-1", "GHSA-xxxx-yyyy-zzzz", "VMSA-2025-0001" }, source: "osv"); + + var clusters = _resolver.Resolve(new[] { vendor, osv }); + + var cluster = Assert.Single(clusters); + Assert.Equal("VMSA-2025-0001", cluster.AdvisoryKey); + Assert.Equal(2, cluster.Advisories.Length); + Assert.True(cluster.Aliases.Any(alias => alias.Value == "VMSA-2025-0001")); + } + + [Fact] + public void Resolve_FallsBackToGhsaWhenOnlyGhsaPresent() + { + var ghsa = CreateAdvisory("GHSA-aaaa-bbbb-cccc", aliases: new[] { "GHSA-aaaa-bbbb-cccc" }, source: "ghsa"); + var osv = CreateAdvisory("OSV-2025-99", aliases: new[] { "OSV-2025-99", "GHSA-aaaa-bbbb-cccc" }, source: "osv"); + + var clusters = _resolver.Resolve(new[] { ghsa, osv }); + + var cluster = Assert.Single(clusters); + Assert.Equal("GHSA-aaaa-bbbb-cccc", cluster.AdvisoryKey); + Assert.Equal(2, cluster.Advisories.Length); + Assert.True(cluster.Aliases.Any(alias => alias.Value == "GHSA-aaaa-bbbb-cccc")); + } + + [Fact] + public void Resolve_GroupsByKeyWhenNoAliases() + { + var first = CreateAdvisory("custom-1", aliases: Array.Empty(), source: "source-a"); + var second = CreateAdvisory("custom-1", aliases: Array.Empty(), source: "source-b"); + + var clusters = _resolver.Resolve(new[] { first, second }); + + var cluster = Assert.Single(clusters); + Assert.Equal("custom-1", cluster.AdvisoryKey); + Assert.Equal(2, cluster.Advisories.Length); + Assert.Contains(cluster.Aliases, alias => alias.Value == "custom-1"); + } + + private static Advisory CreateAdvisory(string key, string[] aliases, string source) + { + var provenance = new[] + { + new AdvisoryProvenance(source, "mapping", key, DateTimeOffset.UtcNow), + }; + + return new Advisory( + key, + $"{key} title", + $"{key} summary", + "en", + DateTimeOffset.UtcNow, + DateTimeOffset.UtcNow, + null, + exploitKnown: false, + aliases, + Array.Empty(), + Array.Empty(), + Array.Empty(), + Array.Empty(), + provenance); + } +} diff --git a/src/StellaOps.Feedser.Merge.Tests/AdvisoryMergeServiceTests.cs b/src/StellaOps.Feedser.Merge.Tests/AdvisoryMergeServiceTests.cs new file mode 100644 index 00000000..b35c31c5 --- /dev/null +++ b/src/StellaOps.Feedser.Merge.Tests/AdvisoryMergeServiceTests.cs @@ -0,0 +1,201 @@ +using System.Collections.Concurrent; +using Microsoft.Extensions.Logging.Abstractions; +using Microsoft.Extensions.Time.Testing; +using StellaOps.Feedser.Core; +using StellaOps.Feedser.Merge.Services; +using StellaOps.Feedser.Models; +using StellaOps.Feedser.Storage.Mongo.Advisories; +using StellaOps.Feedser.Storage.Mongo.Aliases; +using StellaOps.Feedser.Storage.Mongo.MergeEvents; + +namespace StellaOps.Feedser.Merge.Tests; + +public sealed class AdvisoryMergeServiceTests +{ + [Fact] + public async Task MergeAsync_AppliesCanonicalRulesAndPersistsDecisions() + { + var aliasStore = new FakeAliasStore(); + aliasStore.Register("GHSA-aaaa-bbbb-cccc", + (AliasSchemes.Ghsa, "GHSA-aaaa-bbbb-cccc"), + (AliasSchemes.Cve, "CVE-2025-4242")); + aliasStore.Register("CVE-2025-4242", + (AliasSchemes.Cve, "CVE-2025-4242")); + aliasStore.Register("OSV-2025-xyz", + (AliasSchemes.OsV, "OSV-2025-xyz"), + (AliasSchemes.Cve, "CVE-2025-4242")); + + var advisoryStore = new FakeAdvisoryStore(); + advisoryStore.Seed(CreateGhsaAdvisory(), CreateNvdAdvisory(), CreateOsvAdvisory()); + + var mergeEventStore = new InMemoryMergeEventStore(); + var timeProvider = new FakeTimeProvider(new DateTimeOffset(2025, 4, 1, 0, 0, 0, TimeSpan.Zero)); + var writer = new MergeEventWriter(mergeEventStore, new CanonicalHashCalculator(), timeProvider, NullLogger.Instance); + var precedenceMerger = new AdvisoryPrecedenceMerger(new AffectedPackagePrecedenceResolver(), timeProvider); + var aliasResolver = new AliasGraphResolver(aliasStore); + var canonicalMerger = new CanonicalMerger(timeProvider); + var service = new AdvisoryMergeService(aliasResolver, advisoryStore, precedenceMerger, writer, canonicalMerger, NullLogger.Instance); + + var result = await service.MergeAsync("GHSA-aaaa-bbbb-cccc", CancellationToken.None); + + Assert.NotNull(result.Merged); + Assert.Equal("OSV summary overrides", result.Merged!.Summary); + + var upserted = advisoryStore.LastUpserted; + Assert.NotNull(upserted); + Assert.Equal("CVE-2025-4242", upserted!.AdvisoryKey); + Assert.Equal("OSV summary overrides", upserted.Summary); + + var mergeRecord = mergeEventStore.LastRecord; + Assert.NotNull(mergeRecord); + var summaryDecision = Assert.Single(mergeRecord!.FieldDecisions, decision => decision.Field == "summary"); + Assert.Equal("osv", summaryDecision.SelectedSource); + Assert.Equal("freshness_override", summaryDecision.DecisionReason); + } + + private static Advisory CreateGhsaAdvisory() + { + var recorded = DateTimeOffset.Parse("2025-03-01T00:00:00Z"); + var provenance = new AdvisoryProvenance("ghsa", "map", "GHSA-aaaa-bbbb-cccc", recorded, new[] { ProvenanceFieldMasks.Advisory }); + return new Advisory( + "GHSA-aaaa-bbbb-cccc", + "Container escape", + "Initial GHSA summary.", + "en", + recorded, + recorded, + "medium", + exploitKnown: false, + aliases: new[] { "CVE-2025-4242", "GHSA-aaaa-bbbb-cccc" }, + references: Array.Empty(), + affectedPackages: Array.Empty(), + cvssMetrics: Array.Empty(), + provenance: new[] { provenance }); + } + + private static Advisory CreateNvdAdvisory() + { + var recorded = DateTimeOffset.Parse("2025-03-02T00:00:00Z"); + var provenance = new AdvisoryProvenance("nvd", "map", "CVE-2025-4242", recorded, new[] { ProvenanceFieldMasks.Advisory }); + return new Advisory( + "CVE-2025-4242", + "CVE-2025-4242", + "Baseline NVD summary.", + "en", + recorded, + recorded, + "high", + exploitKnown: false, + aliases: new[] { "CVE-2025-4242" }, + references: Array.Empty(), + affectedPackages: Array.Empty(), + cvssMetrics: Array.Empty(), + provenance: new[] { provenance }); + } + + private static Advisory CreateOsvAdvisory() + { + var recorded = DateTimeOffset.Parse("2025-03-05T12:00:00Z"); + var provenance = new AdvisoryProvenance("osv", "map", "OSV-2025-xyz", recorded, new[] { ProvenanceFieldMasks.Advisory }); + return new Advisory( + "OSV-2025-xyz", + "Container escape", + "OSV summary overrides", + "en", + recorded, + recorded, + "critical", + exploitKnown: false, + aliases: new[] { "OSV-2025-xyz", "CVE-2025-4242" }, + references: Array.Empty(), + affectedPackages: Array.Empty(), + cvssMetrics: Array.Empty(), + provenance: new[] { provenance }); + } + + private sealed class FakeAliasStore : IAliasStore + { + private readonly ConcurrentDictionary> _records = new(StringComparer.OrdinalIgnoreCase); + + public void Register(string advisoryKey, params (string Scheme, string Value)[] aliases) + { + var list = new List(); + foreach (var (scheme, value) in aliases) + { + list.Add(new AliasRecord(advisoryKey, scheme, value, DateTimeOffset.UtcNow)); + } + + _records[advisoryKey] = list; + } + + public Task ReplaceAsync(string advisoryKey, IEnumerable aliases, DateTimeOffset updatedAt, CancellationToken cancellationToken) + => Task.FromResult(new AliasUpsertResult(advisoryKey, Array.Empty())); + + public Task> GetByAliasAsync(string scheme, string value, CancellationToken cancellationToken) + { + var matches = _records.Values + .SelectMany(static records => records) + .Where(record => string.Equals(record.Scheme, scheme, StringComparison.OrdinalIgnoreCase) && string.Equals(record.Value, value, StringComparison.OrdinalIgnoreCase)) + .ToList(); + + return Task.FromResult>(matches); + } + + public Task> GetByAdvisoryAsync(string advisoryKey, CancellationToken cancellationToken) + { + if (_records.TryGetValue(advisoryKey, out var records)) + { + return Task.FromResult>(records); + } + + return Task.FromResult>(Array.Empty()); + } + } + + private sealed class FakeAdvisoryStore : IAdvisoryStore + { + private readonly ConcurrentDictionary _advisories = new(StringComparer.OrdinalIgnoreCase); + + public Advisory? LastUpserted { get; private set; } + + public void Seed(params Advisory[] advisories) + { + foreach (var advisory in advisories) + { + _advisories[advisory.AdvisoryKey] = advisory; + } + } + + public Task FindAsync(string advisoryKey, CancellationToken cancellationToken) + { + _advisories.TryGetValue(advisoryKey, out var advisory); + return Task.FromResult(advisory); + } + + public Task> GetRecentAsync(int limit, CancellationToken cancellationToken) + => Task.FromResult>(Array.Empty()); + + public Task UpsertAsync(Advisory advisory, CancellationToken cancellationToken) + { + _advisories[advisory.AdvisoryKey] = advisory; + LastUpserted = advisory; + return Task.CompletedTask; + } + + public IAsyncEnumerable StreamAsync(CancellationToken cancellationToken) => AsyncEnumerable.Empty(); + } + + private sealed class InMemoryMergeEventStore : IMergeEventStore + { + public MergeEventRecord? LastRecord { get; private set; } + + public Task AppendAsync(MergeEventRecord record, CancellationToken cancellationToken) + { + LastRecord = record; + return Task.CompletedTask; + } + + public Task> GetRecentAsync(string advisoryKey, int limit, CancellationToken cancellationToken) + => Task.FromResult>(Array.Empty()); + } +} diff --git a/src/StellaOps.Feedser.Merge.Tests/AdvisoryPrecedenceMergerTests.cs b/src/StellaOps.Feedser.Merge.Tests/AdvisoryPrecedenceMergerTests.cs new file mode 100644 index 00000000..ce64bfac --- /dev/null +++ b/src/StellaOps.Feedser.Merge.Tests/AdvisoryPrecedenceMergerTests.cs @@ -0,0 +1,385 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Time.Testing; +using StellaOps.Feedser.Merge.Options; +using StellaOps.Feedser.Merge.Services; +using StellaOps.Feedser.Models; + +namespace StellaOps.Feedser.Merge.Tests; + +public sealed class AdvisoryPrecedenceMergerTests +{ + [Fact] + public void Merge_PrefersVendorPrecedenceOverNvd() + { + var timeProvider = new FakeTimeProvider(new DateTimeOffset(2025, 1, 1, 0, 0, 0, TimeSpan.Zero)); + var merger = new AdvisoryPrecedenceMerger(new AffectedPackagePrecedenceResolver(), timeProvider); + using var metrics = new MetricCollector("StellaOps.Feedser.Merge"); + + var (redHat, nvd) = CreateVendorAndRegistryAdvisories(); + var expectedMergeTimestamp = timeProvider.GetUtcNow(); + + var merged = merger.Merge(new[] { nvd, redHat }); + + Assert.Equal("CVE-2025-1000", merged.AdvisoryKey); + Assert.Equal("Red Hat Security Advisory", merged.Title); + Assert.Equal("Vendor-confirmed impact on RHEL 9.", merged.Summary); + Assert.Equal("high", merged.Severity); + Assert.Equal(redHat.Published, merged.Published); + Assert.Equal(redHat.Modified, merged.Modified); + Assert.Contains("RHSA-2025:0001", merged.Aliases); + Assert.Contains("CVE-2025-1000", merged.Aliases); + + var package = Assert.Single(merged.AffectedPackages); + Assert.Equal("cpe:2.3:o:redhat:enterprise_linux:9:*:*:*:*:*:*:*", package.Identifier); + Assert.Empty(package.VersionRanges); // NVD range suppressed by vendor precedence + Assert.Contains(package.Statuses, status => status.Status == "known_affected"); + Assert.Contains(package.Provenance, provenance => provenance.Source == "redhat"); + Assert.Contains(package.Provenance, provenance => provenance.Source == "nvd"); + + Assert.Contains(merged.CvssMetrics, metric => metric.Provenance.Source == "redhat"); + Assert.Contains(merged.CvssMetrics, metric => metric.Provenance.Source == "nvd"); + + var mergeProvenance = merged.Provenance.Single(p => p.Source == "merge"); + Assert.Equal("precedence", mergeProvenance.Kind); + Assert.Equal(expectedMergeTimestamp, mergeProvenance.RecordedAt); + Assert.Contains("redhat", mergeProvenance.Value, StringComparison.OrdinalIgnoreCase); + Assert.Contains("nvd", mergeProvenance.Value, StringComparison.OrdinalIgnoreCase); + + var rangeMeasurement = Assert.Single(metrics.Measurements, measurement => measurement.Name == "feedser.merge.range_overrides"); + Assert.Equal(1, rangeMeasurement.Value); + Assert.Contains(rangeMeasurement.Tags, tag => string.Equals(tag.Key, "suppressed_source", StringComparison.Ordinal) && tag.Value?.ToString()?.Contains("nvd", StringComparison.OrdinalIgnoreCase) == true); + + var severityConflict = Assert.Single(metrics.Measurements, measurement => measurement.Name == "feedser.merge.conflicts"); + Assert.Equal(1, severityConflict.Value); + Assert.Contains(severityConflict.Tags, tag => string.Equals(tag.Key, "type", StringComparison.Ordinal) && string.Equals(tag.Value?.ToString(), "severity", StringComparison.OrdinalIgnoreCase)); + } + + [Fact] + public void Merge_KevOnlyTogglesExploitKnown() + { + var timeProvider = new FakeTimeProvider(new DateTimeOffset(2025, 2, 1, 0, 0, 0, TimeSpan.Zero)); + var merger = new AdvisoryPrecedenceMerger(new AffectedPackagePrecedenceResolver(), timeProvider); + + var nvdProvenance = new AdvisoryProvenance("nvd", "document", "https://nvd", timeProvider.GetUtcNow()); + var baseAdvisory = new Advisory( + "CVE-2025-2000", + "CVE-2025-2000", + "Base registry summary", + "en", + new DateTimeOffset(2025, 1, 5, 0, 0, 0, TimeSpan.Zero), + new DateTimeOffset(2025, 1, 6, 0, 0, 0, TimeSpan.Zero), + "medium", + exploitKnown: false, + aliases: new[] { "CVE-2025-2000" }, + credits: Array.Empty(), + references: Array.Empty(), + affectedPackages: new[] + { + new AffectedPackage( + AffectedPackageTypes.Cpe, + "cpe:2.3:a:example:product:2.0:*:*:*:*:*:*:*", + null, + new[] + { + new AffectedVersionRange( + "semver", + "2.0.0", + "2.0.5", + null, + "<2.0.5", + new AdvisoryProvenance("nvd", "cpe_match", "product", timeProvider.GetUtcNow())) + }, + Array.Empty(), + new[] { nvdProvenance }) + }, + cvssMetrics: Array.Empty(), + provenance: new[] { nvdProvenance }); + + var kevProvenance = new AdvisoryProvenance("kev", "catalog", "CVE-2025-2000", timeProvider.GetUtcNow()); + var kevAdvisory = new Advisory( + "CVE-2025-2000", + "Known Exploited Vulnerability", + summary: null, + language: null, + published: null, + modified: null, + severity: null, + exploitKnown: true, + aliases: new[] { "KEV-CVE-2025-2000" }, + credits: Array.Empty(), + references: Array.Empty(), + affectedPackages: Array.Empty(), + cvssMetrics: Array.Empty(), + provenance: new[] { kevProvenance }); + + var merged = merger.Merge(new[] { baseAdvisory, kevAdvisory }); + + Assert.True(merged.ExploitKnown); + Assert.Equal("medium", merged.Severity); // KEV must not override severity + Assert.Equal("Base registry summary", merged.Summary); + Assert.Contains("CVE-2025-2000", merged.Aliases); + Assert.Contains("KEV-CVE-2025-2000", merged.Aliases); + Assert.Contains(merged.Provenance, provenance => provenance.Source == "kev"); + Assert.Contains(merged.Provenance, provenance => provenance.Source == "merge"); + } + + [Fact] + public void Merge_UnionsCreditsFromSources() + { + var timeProvider = new FakeTimeProvider(); + var merger = new AdvisoryPrecedenceMerger(new AffectedPackagePrecedenceResolver(), timeProvider); + + var ghsaCredits = new[] + { + new AdvisoryCredit( + displayName: "maintainer-team", + role: "remediation_developer", + contacts: new[] { "https://github.com/maintainer-team" }, + provenance: new AdvisoryProvenance( + "ghsa", + "credit", + "mantainer-team", + timeProvider.GetUtcNow(), + new[] { ProvenanceFieldMasks.Credits })), + new AdvisoryCredit( + displayName: "security-reporter", + role: "reporter", + contacts: new[] { "https://github.com/security-reporter" }, + provenance: new AdvisoryProvenance( + "ghsa", + "credit", + "security-reporter", + timeProvider.GetUtcNow(), + new[] { ProvenanceFieldMasks.Credits })), + }; + + var ghsa = new Advisory( + "CVE-2025-9000", + "GHSA advisory", + "Reported in GHSA", + "en", + timeProvider.GetUtcNow(), + timeProvider.GetUtcNow(), + "high", + exploitKnown: false, + aliases: new[] { "GHSA-aaaa-bbbb-cccc", "CVE-2025-9000" }, + credits: ghsaCredits, + references: Array.Empty(), + affectedPackages: Array.Empty(), + cvssMetrics: Array.Empty(), + provenance: new[] { new AdvisoryProvenance("ghsa", "document", "https://github.com/advisories/GHSA-aaaa-bbbb-cccc", timeProvider.GetUtcNow(), new[] { ProvenanceFieldMasks.Advisory }) }); + + var osvCredits = new[] + { + new AdvisoryCredit( + displayName: "osv-researcher", + role: "reporter", + contacts: new[] { "mailto:osv-researcher@example.com" }, + provenance: new AdvisoryProvenance( + "osv", + "credit", + "osv-researcher", + timeProvider.GetUtcNow(), + new[] { ProvenanceFieldMasks.Credits })), + new AdvisoryCredit( + displayName: "maintainer-team", + role: "remediation_developer", + contacts: new[] { "https://github.com/maintainer-team" }, + provenance: new AdvisoryProvenance( + "osv", + "credit", + "maintainer-team", + timeProvider.GetUtcNow(), + new[] { ProvenanceFieldMasks.Credits })), + }; + + var osv = new Advisory( + "CVE-2025-9000", + "OSV advisory", + "Reported in OSV.dev", + "en", + timeProvider.GetUtcNow().AddDays(-1), + timeProvider.GetUtcNow().AddHours(-1), + "medium", + exploitKnown: false, + aliases: new[] { "CVE-2025-9000" }, + credits: osvCredits, + references: Array.Empty(), + affectedPackages: Array.Empty(), + cvssMetrics: Array.Empty(), + provenance: new[] { new AdvisoryProvenance("osv", "document", "https://osv.dev/vulnerability/CVE-2025-9000", timeProvider.GetUtcNow(), new[] { ProvenanceFieldMasks.Advisory }) }); + + var merged = merger.Merge(new[] { ghsa, osv }); + + Assert.Equal("CVE-2025-9000", merged.AdvisoryKey); + Assert.Contains(merged.Credits, credit => + string.Equals(credit.DisplayName, "maintainer-team", StringComparison.OrdinalIgnoreCase) && + string.Equals(credit.Role, "remediation_developer", StringComparison.OrdinalIgnoreCase)); + Assert.Contains(merged.Credits, credit => + string.Equals(credit.DisplayName, "osv-researcher", StringComparison.OrdinalIgnoreCase) && + string.Equals(credit.Role, "reporter", StringComparison.OrdinalIgnoreCase)); + Assert.Contains(merged.Credits, credit => + string.Equals(credit.DisplayName, "security-reporter", StringComparison.OrdinalIgnoreCase) && + string.Equals(credit.Role, "reporter", StringComparison.OrdinalIgnoreCase)); + + Assert.Contains(merged.Credits, credit => credit.Provenance.Source == "ghsa"); + Assert.Contains(merged.Credits, credit => credit.Provenance.Source == "osv"); + } + + [Fact] + public void Merge_RespectsConfiguredPrecedenceOverrides() + { + var timeProvider = new FakeTimeProvider(new DateTimeOffset(2025, 3, 1, 0, 0, 0, TimeSpan.Zero)); + var options = new AdvisoryPrecedenceOptions + { + Ranks = new Dictionary(StringComparer.OrdinalIgnoreCase) + { + ["nvd"] = 0, + ["redhat"] = 5, + } + }; + + var logger = new TestLogger(); + using var metrics = new MetricCollector("StellaOps.Feedser.Merge"); + + var merger = new AdvisoryPrecedenceMerger( + new AffectedPackagePrecedenceResolver(), + options, + timeProvider, + logger); + + var (redHat, nvd) = CreateVendorAndRegistryAdvisories(); + var merged = merger.Merge(new[] { redHat, nvd }); + + Assert.Equal("CVE-2025-1000", merged.AdvisoryKey); + Assert.Equal("CVE-2025-1000", merged.Title); // NVD preferred + Assert.Equal("NVD summary", merged.Summary); + Assert.Equal("medium", merged.Severity); + + var package = Assert.Single(merged.AffectedPackages); + Assert.NotEmpty(package.VersionRanges); // Vendor range no longer overrides + Assert.Contains(package.Provenance, provenance => provenance.Source == "nvd"); + Assert.Contains(package.Provenance, provenance => provenance.Source == "redhat"); + + var overrideMeasurement = Assert.Single(metrics.Measurements, m => m.Name == "feedser.merge.overrides"); + Assert.Equal(1, overrideMeasurement.Value); + Assert.Contains(overrideMeasurement.Tags, tag => tag.Key == "primary_source" && string.Equals(tag.Value?.ToString(), "nvd", StringComparison.OrdinalIgnoreCase)); + Assert.Contains(overrideMeasurement.Tags, tag => tag.Key == "suppressed_source" && tag.Value?.ToString()?.Contains("redhat", StringComparison.OrdinalIgnoreCase) == true); + + Assert.DoesNotContain(metrics.Measurements, measurement => measurement.Name == "feedser.merge.range_overrides"); + + var conflictMeasurement = Assert.Single(metrics.Measurements, measurement => measurement.Name == "feedser.merge.conflicts"); + Assert.Equal(1, conflictMeasurement.Value); + Assert.Contains(conflictMeasurement.Tags, tag => tag.Key == "type" && string.Equals(tag.Value?.ToString(), "severity", StringComparison.OrdinalIgnoreCase)); + Assert.Contains(conflictMeasurement.Tags, tag => tag.Key == "reason" && string.Equals(tag.Value?.ToString(), "mismatch", StringComparison.OrdinalIgnoreCase)); + + var logEntry = Assert.Single(logger.Entries, entry => entry.EventId.Name == "AdvisoryOverride"); + Assert.Equal(LogLevel.Information, logEntry.Level); + Assert.NotNull(logEntry.StructuredState); + Assert.Contains(logEntry.StructuredState!, kvp => + (string.Equals(kvp.Key, "Override", StringComparison.Ordinal) || + string.Equals(kvp.Key, "@Override", StringComparison.Ordinal)) && + kvp.Value is not null); + } + + private static (Advisory Vendor, Advisory Registry) CreateVendorAndRegistryAdvisories() + { + var redHatPublished = new DateTimeOffset(2025, 1, 10, 0, 0, 0, TimeSpan.Zero); + var redHatModified = redHatPublished.AddDays(1); + var redHatProvenance = new AdvisoryProvenance("redhat", "advisory", "RHSA-2025:0001", redHatModified); + var redHatPackage = new AffectedPackage( + AffectedPackageTypes.Cpe, + "cpe:2.3:o:redhat:enterprise_linux:9:*:*:*:*:*:*:*", + "rhel-9", + Array.Empty(), + new[] { new AffectedPackageStatus("known_affected", redHatProvenance) }, + new[] { redHatProvenance }); + var redHat = new Advisory( + "CVE-2025-1000", + "Red Hat Security Advisory", + "Vendor-confirmed impact on RHEL 9.", + "en", + redHatPublished, + redHatModified, + "high", + exploitKnown: false, + aliases: new[] { "CVE-2025-1000", "RHSA-2025:0001" }, + credits: Array.Empty(), + references: new[] + { + new AdvisoryReference( + "https://access.redhat.com/errata/RHSA-2025:0001", + "advisory", + "redhat", + "Red Hat errata", + redHatProvenance) + }, + affectedPackages: new[] { redHatPackage }, + cvssMetrics: new[] + { + new CvssMetric( + "3.1", + "CVSS:3.1/AV:N/AC:L/PR:N/UI:N/S:U/C:H/I:H/A:H", + 9.8, + "critical", + new AdvisoryProvenance("redhat", "cvss", "RHSA-2025:0001", redHatModified)) + }, + provenance: new[] { redHatProvenance }); + + var nvdPublished = new DateTimeOffset(2025, 1, 5, 0, 0, 0, TimeSpan.Zero); + var nvdModified = nvdPublished.AddDays(2); + var nvdProvenance = new AdvisoryProvenance("nvd", "document", "https://nvd.nist.gov/vuln/detail/CVE-2025-1000", nvdModified); + var nvdPackage = new AffectedPackage( + AffectedPackageTypes.Cpe, + "cpe:2.3:o:redhat:enterprise_linux:9:*:*:*:*:*:*:*", + "rhel-9", + new[] + { + new AffectedVersionRange( + "cpe", + null, + null, + null, + "<=9.0", + new AdvisoryProvenance("nvd", "cpe_match", "RHEL", nvdModified)) + }, + Array.Empty(), + new[] { nvdProvenance }); + var nvd = new Advisory( + "CVE-2025-1000", + "CVE-2025-1000", + "NVD summary", + "en", + nvdPublished, + nvdModified, + "medium", + exploitKnown: false, + aliases: new[] { "CVE-2025-1000" }, + credits: Array.Empty(), + references: new[] + { + new AdvisoryReference( + "https://nvd.nist.gov/vuln/detail/CVE-2025-1000", + "advisory", + "nvd", + "NVD advisory", + nvdProvenance) + }, + affectedPackages: new[] { nvdPackage }, + cvssMetrics: new[] + { + new CvssMetric( + "3.1", + "CVSS:3.1/AV:N/AC:L/PR:L/UI:R/S:U/C:H/I:H/A:N", + 6.8, + "medium", + new AdvisoryProvenance("nvd", "cvss", "CVE-2025-1000", nvdModified)) + }, + provenance: new[] { nvdProvenance }); + + return (redHat, nvd); + } +} diff --git a/src/StellaOps.Feedser.Merge.Tests/AffectedPackagePrecedenceResolverTests.cs b/src/StellaOps.Feedser.Merge.Tests/AffectedPackagePrecedenceResolverTests.cs new file mode 100644 index 00000000..aa7233eb --- /dev/null +++ b/src/StellaOps.Feedser.Merge.Tests/AffectedPackagePrecedenceResolverTests.cs @@ -0,0 +1,96 @@ +using System; +using StellaOps.Feedser.Merge.Services; +using StellaOps.Feedser.Models; + +namespace StellaOps.Feedser.Merge.Tests; + +public sealed class AffectedPackagePrecedenceResolverTests +{ + [Fact] + public void Merge_PrefersRedHatOverNvdForSameCpe() + { + var redHat = new AffectedPackage( + type: AffectedPackageTypes.Cpe, + identifier: "cpe:2.3:o:redhat:enterprise_linux:9:*:*:*:*:*:*:*", + platform: "RHEL 9", + versionRanges: Array.Empty(), + statuses: new[] + { + new AffectedPackageStatus( + status: "known_affected", + provenance: new AdvisoryProvenance("redhat", "oval", "RHEL-9", DateTimeOffset.Parse("2025-10-01T00:00:00Z"))) + }, + provenance: new[] + { + new AdvisoryProvenance("redhat", "oval", "RHEL-9", DateTimeOffset.Parse("2025-10-01T00:00:00Z")) + }); + + var nvd = new AffectedPackage( + type: AffectedPackageTypes.Cpe, + identifier: "cpe:2.3:o:redhat:enterprise_linux:9:*:*:*:*:*:*:*", + platform: "RHEL 9", + versionRanges: new[] + { + new AffectedVersionRange( + rangeKind: "cpe", + introducedVersion: null, + fixedVersion: null, + lastAffectedVersion: null, + rangeExpression: "<=9.0", + provenance: new AdvisoryProvenance("nvd", "cpe_match", "RHEL-9", DateTimeOffset.Parse("2025-09-30T00:00:00Z"))) + }, + provenance: new[] + { + new AdvisoryProvenance("nvd", "cpe_match", "RHEL-9", DateTimeOffset.Parse("2025-09-30T00:00:00Z")) + }); + + var resolver = new AffectedPackagePrecedenceResolver(); + var result = resolver.Merge(new[] { nvd, redHat }); + + var package = Assert.Single(result.Packages); + Assert.Equal("cpe:2.3:o:redhat:enterprise_linux:9:*:*:*:*:*:*:*", package.Identifier); + Assert.Empty(package.VersionRanges); // NVD range overridden + Assert.Contains(package.Statuses, status => status.Status == "known_affected"); + Assert.Contains(package.Provenance, provenance => provenance.Source == "redhat"); + Assert.Contains(package.Provenance, provenance => provenance.Source == "nvd"); + + var rangeOverride = Assert.Single(result.Overrides); + Assert.Equal("cpe:2.3:o:redhat:enterprise_linux:9:*:*:*:*:*:*:*", rangeOverride.Identifier); + Assert.Equal(0, rangeOverride.PrimaryRank); + Assert.True(rangeOverride.SuppressedRank >= rangeOverride.PrimaryRank); + Assert.Equal(0, rangeOverride.PrimaryRangeCount); + Assert.Equal(1, rangeOverride.SuppressedRangeCount); + } + + [Fact] + public void Merge_KeepsNvdWhenNoHigherPrecedence() + { + var nvd = new AffectedPackage( + type: AffectedPackageTypes.Cpe, + identifier: "cpe:2.3:a:example:product:1.0:*:*:*:*:*:*:*", + platform: null, + versionRanges: new[] + { + new AffectedVersionRange( + rangeKind: "semver", + introducedVersion: null, + fixedVersion: "1.0.1", + lastAffectedVersion: null, + rangeExpression: "<1.0.1", + provenance: new AdvisoryProvenance("nvd", "cpe_match", "product", DateTimeOffset.Parse("2025-09-01T00:00:00Z"))) + }, + provenance: new[] + { + new AdvisoryProvenance("nvd", "cpe_match", "product", DateTimeOffset.Parse("2025-09-01T00:00:00Z")) + }); + + var resolver = new AffectedPackagePrecedenceResolver(); + var result = resolver.Merge(new[] { nvd }); + + var package = Assert.Single(result.Packages); + Assert.Equal(nvd.Identifier, package.Identifier); + Assert.Equal(nvd.VersionRanges.Single().RangeExpression, package.VersionRanges.Single().RangeExpression); + Assert.Equal("nvd", package.Provenance.Single().Source); + Assert.Empty(result.Overrides); + } +} diff --git a/src/StellaOps.Feedser.Merge.Tests/AliasGraphResolverTests.cs b/src/StellaOps.Feedser.Merge.Tests/AliasGraphResolverTests.cs new file mode 100644 index 00000000..6f2542ab --- /dev/null +++ b/src/StellaOps.Feedser.Merge.Tests/AliasGraphResolverTests.cs @@ -0,0 +1,135 @@ +using System; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.Extensions.Logging.Abstractions; +using MongoDB.Driver; +using StellaOps.Feedser.Merge.Services; +using StellaOps.Feedser.Storage.Mongo; +using StellaOps.Feedser.Storage.Mongo.Aliases; +using StellaOps.Feedser.Testing; + +namespace StellaOps.Feedser.Merge.Tests; + +[Collection("mongo-fixture")] +public sealed class AliasGraphResolverTests : IClassFixture +{ + private readonly MongoIntegrationFixture _fixture; + + public AliasGraphResolverTests(MongoIntegrationFixture fixture) + { + _fixture = fixture; + } + + [Fact] + public async Task ResolveAsync_ReturnsCollisions_WhenAliasesOverlap() + { + await DropAliasCollectionAsync(); + + var aliasStore = new AliasStore(_fixture.Database, NullLogger.Instance); + var resolver = new AliasGraphResolver(aliasStore); + + var timestamp = DateTimeOffset.UtcNow; + await aliasStore.ReplaceAsync( + "ADV-1", + new[] { new AliasEntry("CVE", "CVE-2025-2000"), new AliasEntry(AliasStoreConstants.PrimaryScheme, "ADV-1") }, + timestamp, + CancellationToken.None); + + await aliasStore.ReplaceAsync( + "ADV-2", + new[] { new AliasEntry("CVE", "CVE-2025-2000"), new AliasEntry(AliasStoreConstants.PrimaryScheme, "ADV-2") }, + timestamp.AddMinutes(1), + CancellationToken.None); + + var result = await resolver.ResolveAsync("ADV-1", CancellationToken.None); + Assert.NotNull(result); + Assert.Equal("ADV-1", result.AdvisoryKey); + Assert.NotEmpty(result.Collisions); + var collision = Assert.Single(result.Collisions); + Assert.Equal("CVE", collision.Scheme); + Assert.Contains("ADV-1", collision.AdvisoryKeys); + Assert.Contains("ADV-2", collision.AdvisoryKeys); + } + + [Fact] + public async Task BuildComponentAsync_TracesConnectedAdvisories() + { + await DropAliasCollectionAsync(); + var aliasStore = new AliasStore(_fixture.Database, NullLogger.Instance); + var resolver = new AliasGraphResolver(aliasStore); + + var timestamp = DateTimeOffset.UtcNow; + await aliasStore.ReplaceAsync( + "ADV-A", + new[] { new AliasEntry("CVE", "CVE-2025-4000"), new AliasEntry(AliasStoreConstants.PrimaryScheme, "ADV-A") }, + timestamp, + CancellationToken.None); + + await aliasStore.ReplaceAsync( + "ADV-B", + new[] { new AliasEntry("CVE", "CVE-2025-4000"), new AliasEntry(AliasStoreConstants.PrimaryScheme, "ADV-B"), new AliasEntry("OSV", "OSV-2025-1") }, + timestamp.AddMinutes(1), + CancellationToken.None); + + await aliasStore.ReplaceAsync( + "ADV-C", + new[] { new AliasEntry("OSV", "OSV-2025-1"), new AliasEntry(AliasStoreConstants.PrimaryScheme, "ADV-C") }, + timestamp.AddMinutes(2), + CancellationToken.None); + + var component = await resolver.BuildComponentAsync("ADV-A", CancellationToken.None); + Assert.Contains("ADV-A", component.AdvisoryKeys, StringComparer.OrdinalIgnoreCase); + Assert.Contains("ADV-B", component.AdvisoryKeys, StringComparer.OrdinalIgnoreCase); + Assert.Contains("ADV-C", component.AdvisoryKeys, StringComparer.OrdinalIgnoreCase); + Assert.NotEmpty(component.Collisions); + Assert.True(component.AliasMap.ContainsKey("ADV-A")); + Assert.Contains(component.AliasMap["ADV-B"], record => record.Scheme == "OSV" && record.Value == "OSV-2025-1"); + } + + private async Task DropAliasCollectionAsync() + { + try + { + await _fixture.Database.DropCollectionAsync(MongoStorageDefaults.Collections.Alias); + } + catch (MongoDB.Driver.MongoCommandException ex) when (ex.CodeName == "NamespaceNotFound" || ex.Message.Contains("ns not found", StringComparison.OrdinalIgnoreCase)) + { + } + } + + [Fact] + public async Task BuildComponentAsync_LinksOsvAndGhsaAliases() + { + await DropAliasCollectionAsync(); + + var aliasStore = new AliasStore(_fixture.Database, NullLogger.Instance); + var resolver = new AliasGraphResolver(aliasStore); + var timestamp = DateTimeOffset.UtcNow; + + await aliasStore.ReplaceAsync( + "ADV-OSV", + new[] + { + new AliasEntry("OSV", "OSV-2025-2001"), + new AliasEntry("GHSA", "GHSA-zzzz-zzzz-zzzz"), + new AliasEntry(AliasStoreConstants.PrimaryScheme, "ADV-OSV"), + }, + timestamp, + CancellationToken.None); + + await aliasStore.ReplaceAsync( + "ADV-GHSA", + new[] + { + new AliasEntry("GHSA", "GHSA-zzzz-zzzz-zzzz"), + new AliasEntry(AliasStoreConstants.PrimaryScheme, "ADV-GHSA"), + }, + timestamp.AddMinutes(1), + CancellationToken.None); + + var component = await resolver.BuildComponentAsync("ADV-OSV", CancellationToken.None); + + Assert.Contains("ADV-GHSA", component.AdvisoryKeys, StringComparer.OrdinalIgnoreCase); + Assert.Contains(component.Collisions, collision => collision.Scheme == "GHSA" && collision.Value == "GHSA-zzzz-zzzz-zzzz"); + } +} diff --git a/src/StellaOps.Feedser.Merge.Tests/CanonicalHashCalculatorTests.cs b/src/StellaOps.Feedser.Merge.Tests/CanonicalHashCalculatorTests.cs new file mode 100644 index 00000000..93c38494 --- /dev/null +++ b/src/StellaOps.Feedser.Merge.Tests/CanonicalHashCalculatorTests.cs @@ -0,0 +1,86 @@ +using System.Linq; +using StellaOps.Feedser.Merge.Services; +using StellaOps.Feedser.Models; + +namespace StellaOps.Feedser.Merge.Tests; + +public sealed class CanonicalHashCalculatorTests +{ + private static readonly Advisory SampleAdvisory = new( + advisoryKey: "CVE-2024-0001", + title: "Sample advisory", + summary: "A sample summary", + language: "EN", + published: DateTimeOffset.Parse("2024-01-01T00:00:00Z"), + modified: DateTimeOffset.Parse("2024-01-02T00:00:00Z"), + severity: "high", + exploitKnown: true, + aliases: new[] { "GHSA-xyz", "CVE-2024-0001" }, + references: new[] + { + new AdvisoryReference("https://example.com/advisory", "external", "vendor", summary: null, provenance: AdvisoryProvenance.Empty), + new AdvisoryReference("https://example.com/blog", "article", "blog", summary: null, provenance: AdvisoryProvenance.Empty), + }, + affectedPackages: new[] + { + new AffectedPackage( + type: AffectedPackageTypes.SemVer, + identifier: "pkg:npm/sample@1.0.0", + platform: null, + versionRanges: new[] + { + new AffectedVersionRange("semver", "1.0.0", "1.2.0", null, null, AdvisoryProvenance.Empty), + new AffectedVersionRange("semver", "1.2.0", null, null, null, AdvisoryProvenance.Empty), + }, + statuses: Array.Empty(), + provenance: new[] { AdvisoryProvenance.Empty }) + }, + cvssMetrics: new[] + { + new CvssMetric("3.1", "AV:N/AC:L/PR:N/UI:N/S:U/C:H/I:H/A:H", 9.8, "critical", AdvisoryProvenance.Empty) + }, + provenance: new[] { AdvisoryProvenance.Empty }); + + [Fact] + public void ComputeHash_ReturnsDeterministicValue() + { + var calculator = new CanonicalHashCalculator(); + var first = calculator.ComputeHash(SampleAdvisory); + var second = calculator.ComputeHash(SampleAdvisory); + + Assert.Equal(first, second); + } + + [Fact] + public void ComputeHash_IgnoresOrderingDifferences() + { + var calculator = new CanonicalHashCalculator(); + + var reordered = new Advisory( + SampleAdvisory.AdvisoryKey, + SampleAdvisory.Title, + SampleAdvisory.Summary, + SampleAdvisory.Language, + SampleAdvisory.Published, + SampleAdvisory.Modified, + SampleAdvisory.Severity, + SampleAdvisory.ExploitKnown, + aliases: SampleAdvisory.Aliases.Reverse().ToArray(), + references: SampleAdvisory.References.Reverse().ToArray(), + affectedPackages: SampleAdvisory.AffectedPackages.Reverse().ToArray(), + cvssMetrics: SampleAdvisory.CvssMetrics.Reverse().ToArray(), + provenance: SampleAdvisory.Provenance.Reverse().ToArray()); + + var originalHash = calculator.ComputeHash(SampleAdvisory); + var reorderedHash = calculator.ComputeHash(reordered); + + Assert.Equal(originalHash, reorderedHash); + } + + [Fact] + public void ComputeHash_NullReturnsEmpty() + { + var calculator = new CanonicalHashCalculator(); + Assert.Empty(calculator.ComputeHash(null)); + } +} diff --git a/src/StellaOps.Feedser.Merge.Tests/DebianEvrComparerTests.cs b/src/StellaOps.Feedser.Merge.Tests/DebianEvrComparerTests.cs new file mode 100644 index 00000000..91925b68 --- /dev/null +++ b/src/StellaOps.Feedser.Merge.Tests/DebianEvrComparerTests.cs @@ -0,0 +1,84 @@ +using StellaOps.Feedser.Merge.Comparers; +using StellaOps.Feedser.Normalization.Distro; + +namespace StellaOps.Feedser.Merge.Tests; + +public sealed class DebianEvrComparerTests +{ + [Theory] + [InlineData("1:1.2.3-1", 1, "1.2.3", "1")] + [InlineData("1.2.3-1", 0, "1.2.3", "1")] + [InlineData("2:4.5", 2, "4.5", "")] + [InlineData("abc", 0, "abc", "")] + public void TryParse_ReturnsComponents(string input, int expectedEpoch, string expectedVersion, string expectedRevision) + { + var success = DebianEvr.TryParse(input, out var evr); + + Assert.True(success); + Assert.NotNull(evr); + Assert.Equal(expectedEpoch, evr!.Epoch); + Assert.Equal(expectedVersion, evr.Version); + Assert.Equal(expectedRevision, evr.Revision); + Assert.Equal(input, evr.Original); + } + + [Theory] + [InlineData("")] + [InlineData(":1.0-1")] + [InlineData("1:")] + public void TryParse_InvalidInputs_ReturnFalse(string input) + { + var success = DebianEvr.TryParse(input, out var evr); + + Assert.False(success); + Assert.Null(evr); + } + + [Fact] + public void Compare_PrefersHigherEpoch() + { + var lower = "0:2.0-1"; + var higher = "1:1.0-1"; + + Assert.True(DebianEvrComparer.Instance.Compare(higher, lower) > 0); + } + + [Fact] + public void Compare_UsesVersionOrdering() + { + var lower = "0:1.2.3-1"; + var higher = "0:1.10.0-1"; + + Assert.True(DebianEvrComparer.Instance.Compare(higher, lower) > 0); + } + + [Fact] + public void Compare_TildeRanksEarlier() + { + var prerelease = "0:1.0~beta1-1"; + var stable = "0:1.0-1"; + + Assert.True(DebianEvrComparer.Instance.Compare(prerelease, stable) < 0); + } + + [Fact] + public void Compare_RevisionBreaksTies() + { + var first = "0:1.0-1"; + var second = "0:1.0-2"; + + Assert.True(DebianEvrComparer.Instance.Compare(second, first) > 0); + } + + [Fact] + public void Compare_FallsBackToOrdinalForInvalid() + { + var left = "not-an-evr"; + var right = "also-not"; + + var expected = Math.Sign(string.CompareOrdinal(left, right)); + var actual = Math.Sign(DebianEvrComparer.Instance.Compare(left, right)); + + Assert.Equal(expected, actual); + } +} diff --git a/src/StellaOps.Feedser.Merge.Tests/MergeEventWriterTests.cs b/src/StellaOps.Feedser.Merge.Tests/MergeEventWriterTests.cs new file mode 100644 index 00000000..9b6f6f5b --- /dev/null +++ b/src/StellaOps.Feedser.Merge.Tests/MergeEventWriterTests.cs @@ -0,0 +1,85 @@ +using Microsoft.Extensions.Logging.Abstractions; +using Microsoft.Extensions.Time.Testing; +using StellaOps.Feedser.Merge.Services; +using StellaOps.Feedser.Models; +using StellaOps.Feedser.Storage.Mongo.MergeEvents; + +namespace StellaOps.Feedser.Merge.Tests; + +public sealed class MergeEventWriterTests +{ + [Fact] + public async Task AppendAsync_WritesRecordWithComputedHashes() + { + var store = new InMemoryMergeEventStore(); + var calculator = new CanonicalHashCalculator(); + var timeProvider = new FakeTimeProvider(DateTimeOffset.Parse("2024-05-01T00:00:00Z")); + var writer = new MergeEventWriter(store, calculator, timeProvider, NullLogger.Instance); + + var before = CreateAdvisory("CVE-2024-0001", "Initial"); + var after = CreateAdvisory("CVE-2024-0001", "Sample", summary: "Updated"); + + var documentIds = new[] { Guid.NewGuid(), Guid.NewGuid() }; + var record = await writer.AppendAsync("CVE-2024-0001", before, after, documentIds, Array.Empty(), CancellationToken.None); + + Assert.NotEqual(Guid.Empty, record.Id); + Assert.Equal("CVE-2024-0001", record.AdvisoryKey); + Assert.True(record.AfterHash.Length > 0); + Assert.Equal(timeProvider.GetUtcNow(), record.MergedAt); + Assert.Equal(documentIds, record.InputDocumentIds); + Assert.NotNull(store.LastRecord); + Assert.Same(store.LastRecord, record); + } + + [Fact] + public async Task AppendAsync_NullBeforeUsesEmptyHash() + { + var store = new InMemoryMergeEventStore(); + var calculator = new CanonicalHashCalculator(); + var timeProvider = new FakeTimeProvider(DateTimeOffset.Parse("2024-05-01T00:00:00Z")); + var writer = new MergeEventWriter(store, calculator, timeProvider, NullLogger.Instance); + + var after = CreateAdvisory("CVE-2024-0002", "Changed"); + + var record = await writer.AppendAsync("CVE-2024-0002", null, after, Array.Empty(), Array.Empty(), CancellationToken.None); + + Assert.Empty(record.BeforeHash); + Assert.True(record.AfterHash.Length > 0); + } + + + private static Advisory CreateAdvisory(string advisoryKey, string title, string? summary = null) + { + return new Advisory( + advisoryKey, + title, + summary, + language: "en", + published: DateTimeOffset.Parse("2024-01-01T00:00:00Z"), + modified: DateTimeOffset.Parse("2024-01-02T00:00:00Z"), + severity: "medium", + exploitKnown: false, + aliases: new[] { advisoryKey }, + references: new[] + { + new AdvisoryReference("https://example.com/" + advisoryKey.ToLowerInvariant(), "external", "vendor", summary: null, provenance: AdvisoryProvenance.Empty) + }, + affectedPackages: Array.Empty(), + cvssMetrics: Array.Empty(), + provenance: Array.Empty()); + } + + private sealed class InMemoryMergeEventStore : IMergeEventStore + { + public MergeEventRecord? LastRecord { get; private set; } + + public Task AppendAsync(MergeEventRecord record, CancellationToken cancellationToken) + { + LastRecord = record; + return Task.CompletedTask; + } + + public Task> GetRecentAsync(string advisoryKey, int limit, CancellationToken cancellationToken) + => Task.FromResult>(Array.Empty()); + } +} diff --git a/src/StellaOps.Feedser.Merge.Tests/MergePrecedenceIntegrationTests.cs b/src/StellaOps.Feedser.Merge.Tests/MergePrecedenceIntegrationTests.cs new file mode 100644 index 00000000..1d2fca32 --- /dev/null +++ b/src/StellaOps.Feedser.Merge.Tests/MergePrecedenceIntegrationTests.cs @@ -0,0 +1,231 @@ +using System; +using System.Linq; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.Extensions.Logging.Abstractions; +using Microsoft.Extensions.Time.Testing; +using MongoDB.Driver; +using StellaOps.Feedser.Merge.Services; +using StellaOps.Feedser.Models; +using StellaOps.Feedser.Storage.Mongo; +using StellaOps.Feedser.Storage.Mongo.MergeEvents; +using StellaOps.Feedser.Testing; + +namespace StellaOps.Feedser.Merge.Tests; + +[Collection("mongo-fixture")] +public sealed class MergePrecedenceIntegrationTests : IAsyncLifetime +{ + private readonly MongoIntegrationFixture _fixture; + private MergeEventStore? _mergeEventStore; + private MergeEventWriter? _mergeEventWriter; + private AdvisoryPrecedenceMerger? _merger; + private FakeTimeProvider? _timeProvider; + + public MergePrecedenceIntegrationTests(MongoIntegrationFixture fixture) + { + _fixture = fixture; + } + + [Fact] + public async Task MergePipeline_PsirtOverridesNvd_AndKevOnlyTogglesExploitKnown() + { + await EnsureInitializedAsync(); + + var merger = _merger!; + var writer = _mergeEventWriter!; + var store = _mergeEventStore!; + var timeProvider = _timeProvider!; + + var expectedTimestamp = timeProvider.GetUtcNow(); + + var nvd = CreateNvdBaseline(); + var vendor = CreateVendorOverride(); + var kev = CreateKevSignal(); + + var merged = merger.Merge(new[] { nvd, vendor, kev }); + + Assert.Equal("CVE-2025-1000", merged.AdvisoryKey); + Assert.Equal("Vendor Security Advisory", merged.Title); + Assert.Equal("Critical impact on supported platforms.", merged.Summary); + Assert.Equal("critical", merged.Severity); + Assert.True(merged.ExploitKnown); + + var affected = Assert.Single(merged.AffectedPackages); + Assert.Empty(affected.VersionRanges); + Assert.Contains(affected.Statuses, status => status.Status == "known_affected" && status.Provenance.Source == "vendor"); + + var mergeProvenance = Assert.Single(merged.Provenance, p => p.Source == "merge"); + Assert.Equal("precedence", mergeProvenance.Kind); + Assert.Equal(expectedTimestamp, mergeProvenance.RecordedAt); + Assert.Contains("vendor", mergeProvenance.Value, StringComparison.OrdinalIgnoreCase); + Assert.Contains("kev", mergeProvenance.Value, StringComparison.OrdinalIgnoreCase); + + var inputDocumentIds = new[] { Guid.NewGuid(), Guid.NewGuid(), Guid.NewGuid() }; + var record = await writer.AppendAsync(merged.AdvisoryKey, nvd, merged, inputDocumentIds, Array.Empty(), CancellationToken.None); + + Assert.Equal(expectedTimestamp, record.MergedAt); + Assert.Equal(inputDocumentIds, record.InputDocumentIds); + Assert.NotEqual(record.BeforeHash, record.AfterHash); + + var records = await store.GetRecentAsync(merged.AdvisoryKey, 5, CancellationToken.None); + var persisted = Assert.Single(records); + Assert.Equal(record.Id, persisted.Id); + Assert.Equal(merged.AdvisoryKey, persisted.AdvisoryKey); + Assert.True(persisted.AfterHash.Length > 0); + Assert.True(persisted.BeforeHash.Length > 0); + } + + [Fact] + public async Task MergePipeline_IsDeterministicAcrossRuns() + { + await EnsureInitializedAsync(); + + var merger = _merger!; + var calculator = new CanonicalHashCalculator(); + + var first = merger.Merge(new[] { CreateNvdBaseline(), CreateVendorOverride() }); + var second = merger.Merge(new[] { CreateNvdBaseline(), CreateVendorOverride() }); + + var firstHash = calculator.ComputeHash(first); + var secondHash = calculator.ComputeHash(second); + + Assert.Equal(firstHash, secondHash); + Assert.Equal(first.AdvisoryKey, second.AdvisoryKey); + Assert.Equal(first.Aliases.Length, second.Aliases.Length); + Assert.True(first.Aliases.SequenceEqual(second.Aliases)); + } + + public async Task InitializeAsync() + { + _timeProvider = new FakeTimeProvider(new DateTimeOffset(2025, 3, 1, 0, 0, 0, TimeSpan.Zero)) + { + AutoAdvanceAmount = TimeSpan.Zero, + }; + _merger = new AdvisoryPrecedenceMerger(new AffectedPackagePrecedenceResolver(), _timeProvider); + _mergeEventStore = new MergeEventStore(_fixture.Database, NullLogger.Instance); + _mergeEventWriter = new MergeEventWriter(_mergeEventStore, new CanonicalHashCalculator(), _timeProvider, NullLogger.Instance); + await DropMergeCollectionAsync(); + } + + public Task DisposeAsync() => Task.CompletedTask; + + private async Task EnsureInitializedAsync() + { + if (_mergeEventWriter is null) + { + await InitializeAsync(); + } + } + + private async Task DropMergeCollectionAsync() + { + try + { + await _fixture.Database.DropCollectionAsync(MongoStorageDefaults.Collections.MergeEvent); + } + catch (MongoCommandException ex) when (ex.CodeName == "NamespaceNotFound" || ex.Message.Contains("ns not found", StringComparison.OrdinalIgnoreCase)) + { + // Collection has not been created yet – safe to ignore. + } + } + + private static Advisory CreateNvdBaseline() + { + var provenance = new AdvisoryProvenance("nvd", "document", "https://nvd.nist.gov/vuln/detail/CVE-2025-1000", DateTimeOffset.Parse("2025-02-10T00:00:00Z")); + return new Advisory( + "CVE-2025-1000", + "CVE-2025-1000", + "Baseline description from NVD.", + "en", + DateTimeOffset.Parse("2025-02-05T00:00:00Z"), + DateTimeOffset.Parse("2025-02-10T12:00:00Z"), + "medium", + exploitKnown: false, + aliases: new[] { "CVE-2025-1000" }, + references: new[] + { + new AdvisoryReference("https://nvd.nist.gov/vuln/detail/CVE-2025-1000", "advisory", "nvd", "NVD reference", provenance), + }, + affectedPackages: new[] + { + new AffectedPackage( + AffectedPackageTypes.Cpe, + "cpe:2.3:o:vendor:product:1.0:*:*:*:*:*:*:*", + "vendor-os", + new[] + { + new AffectedVersionRange( + rangeKind: "cpe", + introducedVersion: null, + fixedVersion: null, + lastAffectedVersion: null, + rangeExpression: "<=1.0", + provenance: provenance) + }, + Array.Empty(), + new[] { provenance }) + }, + cvssMetrics: new[] + { + new CvssMetric("3.1", "CVSS:3.1/AV:N/AC:L/PR:N/UI:N/S:U/C:H/I:H/A:H", 9.8, "critical", provenance) + }, + provenance: new[] { provenance }); + } + + private static Advisory CreateVendorOverride() + { + var provenance = new AdvisoryProvenance("vendor", "psirt", "VSA-2025-1000", DateTimeOffset.Parse("2025-02-11T00:00:00Z")); + return new Advisory( + "CVE-2025-1000", + "Vendor Security Advisory", + "Critical impact on supported platforms.", + "en", + DateTimeOffset.Parse("2025-02-06T00:00:00Z"), + DateTimeOffset.Parse("2025-02-11T06:00:00Z"), + "critical", + exploitKnown: false, + aliases: new[] { "CVE-2025-1000", "VSA-2025-1000" }, + references: new[] + { + new AdvisoryReference("https://vendor.example/advisories/VSA-2025-1000", "advisory", "vendor", "Vendor advisory", provenance), + }, + affectedPackages: new[] + { + new AffectedPackage( + AffectedPackageTypes.Cpe, + "cpe:2.3:o:vendor:product:1.0:*:*:*:*:*:*:*", + "vendor-os", + Array.Empty(), + new[] + { + new AffectedPackageStatus("known_affected", provenance) + }, + new[] { provenance }) + }, + cvssMetrics: new[] + { + new CvssMetric("3.1", "CVSS:3.1/AV:N/AC:L/PR:N/UI:N/S:C/C:H/I:H/A:H", 10.0, "critical", provenance) + }, + provenance: new[] { provenance }); + } + + private static Advisory CreateKevSignal() + { + var provenance = new AdvisoryProvenance("kev", "catalog", "CVE-2025-1000", DateTimeOffset.Parse("2025-02-12T00:00:00Z")); + return new Advisory( + "CVE-2025-1000", + "Known Exploited Vulnerability", + null, + null, + published: null, + modified: null, + severity: null, + exploitKnown: true, + aliases: new[] { "KEV-CVE-2025-1000" }, + references: Array.Empty(), + affectedPackages: Array.Empty(), + cvssMetrics: Array.Empty(), + provenance: new[] { provenance }); + } +} diff --git a/src/StellaOps.Feedser.Merge.Tests/MetricCollector.cs b/src/StellaOps.Feedser.Merge.Tests/MetricCollector.cs new file mode 100644 index 00000000..36e3bde2 --- /dev/null +++ b/src/StellaOps.Feedser.Merge.Tests/MetricCollector.cs @@ -0,0 +1,56 @@ +using System; +using System.Collections.Generic; +using System.Diagnostics.Metrics; +using System.Linq; + +namespace StellaOps.Feedser.Merge.Tests; + +internal sealed class MetricCollector : IDisposable +{ + private readonly MeterListener _listener; + private readonly List _measurements = new(); + + public MetricCollector(string meterName) + { + if (string.IsNullOrWhiteSpace(meterName)) + { + throw new ArgumentException("Meter name is required", nameof(meterName)); + } + + _listener = new MeterListener + { + InstrumentPublished = (instrument, listener) => + { + if (instrument.Meter.Name == meterName) + { + listener.EnableMeasurementEvents(instrument); + } + } + }; + + _listener.SetMeasurementEventCallback((instrument, measurement, tags, state) => + { + var tagArray = new KeyValuePair[tags.Length]; + for (var i = 0; i < tags.Length; i++) + { + tagArray[i] = tags[i]; + } + + _measurements.Add(new MetricMeasurement(instrument.Name, measurement, tagArray)); + }); + + _listener.Start(); + } + + public IReadOnlyList Measurements => _measurements; + + public void Dispose() + { + _listener.Dispose(); + } + + internal sealed record MetricMeasurement( + string Name, + long Value, + IReadOnlyList> Tags); +} diff --git a/src/StellaOps.Feedser.Merge.Tests/NevraComparerTests.cs b/src/StellaOps.Feedser.Merge.Tests/NevraComparerTests.cs new file mode 100644 index 00000000..2dbbe53e --- /dev/null +++ b/src/StellaOps.Feedser.Merge.Tests/NevraComparerTests.cs @@ -0,0 +1,108 @@ +using StellaOps.Feedser.Merge.Comparers; +using StellaOps.Feedser.Normalization.Distro; + +namespace StellaOps.Feedser.Merge.Tests; + +public sealed class NevraComparerTests +{ + [Theory] + [InlineData("kernel-1:4.18.0-348.7.1.el8_5.x86_64", "kernel", 1, "4.18.0", "348.7.1.el8_5", "x86_64")] + [InlineData("bash-5.1.8-2.fc35.x86_64", "bash", 0, "5.1.8", "2.fc35", "x86_64")] + [InlineData("openssl-libs-1:1.1.1k-7.el8", "openssl-libs", 1, "1.1.1k", "7.el8", null)] + [InlineData("java-11-openjdk-1:11.0.23.0.9-2.el9_4.ppc64le", "java-11-openjdk", 1, "11.0.23.0.9", "2.el9_4", "ppc64le")] + [InlineData("bash-0:5.2.15-3.el9_4.arm64", "bash", 0, "5.2.15", "3.el9_4", "arm64")] + [InlineData("podman-3:4.9.3-1.el9.x86_64", "podman", 3, "4.9.3", "1.el9", "x86_64")] + public void TryParse_ReturnsExpectedComponents(string input, string expectedName, int expectedEpoch, string expectedVersion, string expectedRelease, string? expectedArch) + { + var success = Nevra.TryParse(input, out var nevra); + + Assert.True(success); + Assert.NotNull(nevra); + Assert.Equal(expectedName, nevra!.Name); + Assert.Equal(expectedEpoch, nevra.Epoch); + Assert.Equal(expectedVersion, nevra.Version); + Assert.Equal(expectedRelease, nevra.Release); + Assert.Equal(expectedArch, nevra.Architecture); + Assert.Equal(input, nevra.Original); + } + + [Theory] + [InlineData("")] + [InlineData("kernel4.18.0-80.el8")] + [InlineData("kernel-4.18.0")] + public void TryParse_InvalidInputs_ReturnFalse(string input) + { + var success = Nevra.TryParse(input, out var nevra); + + Assert.False(success); + Assert.Null(nevra); + } + + [Fact] + public void TryParse_TrimsWhitespace() + { + var success = Nevra.TryParse(" kernel-0:4.18.0-80.el8.x86_64 ", out var nevra); + + Assert.True(success); + Assert.NotNull(nevra); + Assert.Equal("kernel", nevra!.Name); + Assert.Equal("4.18.0", nevra.Version); + } + + [Fact] + public void Compare_PrefersHigherEpoch() + { + var older = "kernel-0:4.18.0-348.7.1.el8_5.x86_64"; + var newer = "kernel-1:4.18.0-348.7.1.el8_5.x86_64"; + + Assert.True(NevraComparer.Instance.Compare(newer, older) > 0); + Assert.True(NevraComparer.Instance.Compare(older, newer) < 0); + } + + [Fact] + public void Compare_UsesRpmVersionOrdering() + { + var lower = "kernel-0:4.18.0-80.el8.x86_64"; + var higher = "kernel-0:4.18.11-80.el8.x86_64"; + + Assert.True(NevraComparer.Instance.Compare(higher, lower) > 0); + } + + [Fact] + public void Compare_UsesReleaseOrdering() + { + var el8 = "bash-0:5.1.0-1.el8.x86_64"; + var el9 = "bash-0:5.1.0-1.el9.x86_64"; + + Assert.True(NevraComparer.Instance.Compare(el9, el8) > 0); + } + + [Fact] + public void Compare_TildeRanksEarlier() + { + var prerelease = "bash-0:5.1.0~beta-1.fc34.x86_64"; + var stable = "bash-0:5.1.0-1.fc34.x86_64"; + + Assert.True(NevraComparer.Instance.Compare(prerelease, stable) < 0); + } + + [Fact] + public void Compare_ConsidersArchitecture() + { + var noarch = "pkg-0:1.0-1.noarch"; + var arch = "pkg-0:1.0-1.x86_64"; + + Assert.True(NevraComparer.Instance.Compare(noarch, arch) < 0); + } + + [Fact] + public void Compare_FallsBackToOrdinalForInvalid() + { + var left = "not-a-nevra"; + var right = "also-not"; + + var expected = Math.Sign(string.CompareOrdinal(left, right)); + var actual = Math.Sign(NevraComparer.Instance.Compare(left, right)); + Assert.Equal(expected, actual); + } +} diff --git a/src/StellaOps.Feedser.Merge.Tests/SemanticVersionRangeResolverTests.cs b/src/StellaOps.Feedser.Merge.Tests/SemanticVersionRangeResolverTests.cs new file mode 100644 index 00000000..3937575a --- /dev/null +++ b/src/StellaOps.Feedser.Merge.Tests/SemanticVersionRangeResolverTests.cs @@ -0,0 +1,67 @@ +using StellaOps.Feedser.Merge.Comparers; + +namespace StellaOps.Feedser.Merge.Tests; + +public sealed class SemanticVersionRangeResolverTests +{ + [Theory] + [InlineData("1.2.3", true)] + [InlineData("1.2.3-beta.1", true)] + [InlineData("invalid", false)] + [InlineData(null, false)] + public void TryParse_ReturnsExpected(string? input, bool expected) + { + var success = SemanticVersionRangeResolver.TryParse(input, out var version); + + Assert.Equal(expected, success); + Assert.Equal(expected, version is not null); + } + + [Fact] + public void Compare_ParsesSemanticVersions() + { + Assert.True(SemanticVersionRangeResolver.Compare("1.2.3", "1.2.2") > 0); + Assert.True(SemanticVersionRangeResolver.Compare("1.2.3-beta", "1.2.3") < 0); + } + + [Fact] + public void Compare_UsesOrdinalFallbackForInvalid() + { + var left = "zzz"; + var right = "aaa"; + var expected = Math.Sign(string.CompareOrdinal(left, right)); + var actual = Math.Sign(SemanticVersionRangeResolver.Compare(left, right)); + + Assert.Equal(expected, actual); + } + + [Fact] + public void ResolveWindows_WithFixedVersion_ComputesExclusiveUpper() + { + var (introduced, exclusive, inclusive) = SemanticVersionRangeResolver.ResolveWindows("1.0.0", "1.2.0", null); + + Assert.Equal(SemanticVersionRangeResolver.Parse("1.0.0"), introduced); + Assert.Equal(SemanticVersionRangeResolver.Parse("1.2.0"), exclusive); + Assert.Null(inclusive); + } + + [Fact] + public void ResolveWindows_WithLastAffectedOnly_ComputesInclusiveAndExclusive() + { + var (introduced, exclusive, inclusive) = SemanticVersionRangeResolver.ResolveWindows("1.0.0", null, "1.1.5"); + + Assert.Equal(SemanticVersionRangeResolver.Parse("1.0.0"), introduced); + Assert.Equal(SemanticVersionRangeResolver.Parse("1.1.6"), exclusive); + Assert.Equal(SemanticVersionRangeResolver.Parse("1.1.5"), inclusive); + } + + [Fact] + public void ResolveWindows_WithNeither_ReturnsNullBounds() + { + var (introduced, exclusive, inclusive) = SemanticVersionRangeResolver.ResolveWindows(null, null, null); + + Assert.Null(introduced); + Assert.Null(exclusive); + Assert.Null(inclusive); + } +} diff --git a/src/StellaOps.Feedser.Merge.Tests/StellaOps.Feedser.Merge.Tests.csproj b/src/StellaOps.Feedser.Merge.Tests/StellaOps.Feedser.Merge.Tests.csproj new file mode 100644 index 00000000..756df80f --- /dev/null +++ b/src/StellaOps.Feedser.Merge.Tests/StellaOps.Feedser.Merge.Tests.csproj @@ -0,0 +1,13 @@ + + + net10.0 + enable + enable + + + + + + + + diff --git a/src/StellaOps.Feedser.Merge.Tests/TestLogger.cs b/src/StellaOps.Feedser.Merge.Tests/TestLogger.cs new file mode 100644 index 00000000..5f42c8f3 --- /dev/null +++ b/src/StellaOps.Feedser.Merge.Tests/TestLogger.cs @@ -0,0 +1,52 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using Microsoft.Extensions.Logging; + +namespace StellaOps.Feedser.Merge.Tests; + +internal sealed class TestLogger : ILogger +{ + private static readonly IDisposable NoopScope = new DisposableScope(); + + public List Entries { get; } = new(); + + public IDisposable BeginScope(TState state) + where TState : notnull + => NoopScope; + + public bool IsEnabled(LogLevel logLevel) => true; + + public void Log(LogLevel logLevel, EventId eventId, TState state, Exception? exception, Func formatter) + { + if (formatter is null) + { + throw new ArgumentNullException(nameof(formatter)); + } + + IReadOnlyList>? structuredState = null; + if (state is IReadOnlyList> list) + { + structuredState = list.ToArray(); + } + else if (state is IEnumerable> enumerable) + { + structuredState = enumerable.ToArray(); + } + + Entries.Add(new LogEntry(logLevel, eventId, formatter(state, exception), structuredState)); + } + + internal sealed record LogEntry( + LogLevel Level, + EventId EventId, + string Message, + IReadOnlyList>? StructuredState); + + private sealed class DisposableScope : IDisposable + { + public void Dispose() + { + } + } +} diff --git a/src/StellaOps.Feedser.Merge/AGENTS.md b/src/StellaOps.Feedser.Merge/AGENTS.md new file mode 100644 index 00000000..ba7327f7 --- /dev/null +++ b/src/StellaOps.Feedser.Merge/AGENTS.md @@ -0,0 +1,33 @@ +# AGENTS +## Role +Deterministic merge and reconciliation engine; builds identity graph via aliases; applies precedence (PSIRT/OVAL > NVD; KEV flag only; regional feeds enrich); produces canonical advisory JSON and merge_event audit trail. +## Scope +- Identity: resolve advisory_key (prefer CVE, else PSIRT/Distro/JVN/BDU/GHSA/ICSA); unify aliases; detect collisions. +- Precedence: override rules for affected ranges (vendor PSIRT/OVAL over registry), enrichment-only feeds (CERTs/JVN/RU-CERT), KEV toggles exploitKnown only. +- Range comparers: RPM NEVRA comparer (epoch:version-release), Debian EVR comparer, SemVer range resolver; platform-aware selection. +- Merge algorithm: stable ordering, pure functions, idempotence; compute beforeHash/afterHash over canonical form; write merge_event. +- Conflict reporting: counters and logs for identity conflicts, reference merges, range overrides. +## Participants +- Storage.Mongo (reads raw mapped advisories, writes merged docs plus merge_event). +- Models (canonical types). +- Exporters (consume merged canonical). +- Core/WebService (jobs: merge:run, maybe per-kind). +## Interfaces & contracts +- AdvisoryMergeService.MergeAsync(ids or byKind): returns summary {processed, merged, overrides, conflicts}. +- Precedence table configurable but with sane defaults: RedHat/Ubuntu/Debian/SUSE > Vendor PSIRT > GHSA/OSV > NVD; CERTs enrich; KEV sets flags. +- Range selection uses comparers: NevraComparer, DebEvrComparer, SemVerRange; deterministic tie-breakers. +- Provenance propagation merges unique entries; references deduped by (url, type). + +## Configuration +- Precedence overrides bind via `feedser:merge:precedence:ranks` (dictionary of `source` → `rank`, lower wins). Absent entries fall back to defaults. +- Operator workflow: update `etc/feedser.yaml` or environment variables, restart merge job; overrides surface in metrics/logs as `AdvisoryOverride` entries. +## In/Out of scope +In: merge logic, precedence policy, hashing, event records, comparers. +Out: fetching/parsing, exporter packaging, signing. +## Observability & security expectations +- Metrics: merge.delta.count, merge.identity.conflicts, merge.range.overrides, merge.duration_ms. +- Logs: decisions (why replaced), keys involved, hashes; avoid dumping large blobs; redact secrets (none expected). +## Tests +- Author and review coverage in `../StellaOps.Feedser.Merge.Tests`. +- Shared fixtures (e.g., `MongoIntegrationFixture`, `ConnectorTestHarness`) live in `../StellaOps.Feedser.Testing`. +- Keep fixtures deterministic; match new cases to real-world advisories or regression scenarios. diff --git a/src/StellaOps.Feedser.Merge/Class1.cs b/src/StellaOps.Feedser.Merge/Class1.cs new file mode 100644 index 00000000..cf4a342f --- /dev/null +++ b/src/StellaOps.Feedser.Merge/Class1.cs @@ -0,0 +1 @@ +// Intentionally left blank; types moved into dedicated files. diff --git a/src/StellaOps.Feedser.Merge/Comparers/DebianEvr.cs b/src/StellaOps.Feedser.Merge/Comparers/DebianEvr.cs new file mode 100644 index 00000000..f478cbc0 --- /dev/null +++ b/src/StellaOps.Feedser.Merge/Comparers/DebianEvr.cs @@ -0,0 +1,232 @@ +namespace StellaOps.Feedser.Merge.Comparers; + +using System; +using StellaOps.Feedser.Normalization.Distro; + +public sealed class DebianEvrComparer : IComparer, IComparer +{ + public static DebianEvrComparer Instance { get; } = new(); + + private DebianEvrComparer() + { + } + + public int Compare(string? x, string? y) + { + if (ReferenceEquals(x, y)) + { + return 0; + } + + if (x is null) + { + return -1; + } + + if (y is null) + { + return 1; + } + + var xParsed = DebianEvr.TryParse(x, out var xEvr); + var yParsed = DebianEvr.TryParse(y, out var yEvr); + + if (xParsed && yParsed) + { + return Compare(xEvr, yEvr); + } + + if (xParsed) + { + return 1; + } + + if (yParsed) + { + return -1; + } + + return string.Compare(x, y, StringComparison.Ordinal); + } + + public int Compare(DebianEvr? x, DebianEvr? y) + { + if (ReferenceEquals(x, y)) + { + return 0; + } + + if (x is null) + { + return -1; + } + + if (y is null) + { + return 1; + } + + var compare = x.Epoch.CompareTo(y.Epoch); + if (compare != 0) + { + return compare; + } + + compare = CompareSegment(x.Version, y.Version); + if (compare != 0) + { + return compare; + } + + compare = CompareSegment(x.Revision, y.Revision); + if (compare != 0) + { + return compare; + } + + return string.Compare(x.Original, y.Original, StringComparison.Ordinal); + } + + private static int CompareSegment(string left, string right) + { + var i = 0; + var j = 0; + + while (i < left.Length || j < right.Length) + { + while (i < left.Length && !IsAlphaNumeric(left[i]) && left[i] != '~') + { + i++; + } + + while (j < right.Length && !IsAlphaNumeric(right[j]) && right[j] != '~') + { + j++; + } + + var leftChar = i < left.Length ? left[i] : '\0'; + var rightChar = j < right.Length ? right[j] : '\0'; + + if (leftChar == '~' || rightChar == '~') + { + if (leftChar != rightChar) + { + return leftChar == '~' ? -1 : 1; + } + + i += leftChar == '~' ? 1 : 0; + j += rightChar == '~' ? 1 : 0; + continue; + } + + var leftIsDigit = char.IsDigit(leftChar); + var rightIsDigit = char.IsDigit(rightChar); + + if (leftIsDigit && rightIsDigit) + { + var leftStart = i; + while (i < left.Length && char.IsDigit(left[i])) + { + i++; + } + + var rightStart = j; + while (j < right.Length && char.IsDigit(right[j])) + { + j++; + } + + var leftTrimmed = leftStart; + while (leftTrimmed < i && left[leftTrimmed] == '0') + { + leftTrimmed++; + } + + var rightTrimmed = rightStart; + while (rightTrimmed < j && right[rightTrimmed] == '0') + { + rightTrimmed++; + } + + var leftLength = i - leftTrimmed; + var rightLength = j - rightTrimmed; + + if (leftLength != rightLength) + { + return leftLength.CompareTo(rightLength); + } + + var comparison = left.AsSpan(leftTrimmed, leftLength) + .CompareTo(right.AsSpan(rightTrimmed, rightLength), StringComparison.Ordinal); + if (comparison != 0) + { + return comparison; + } + + continue; + } + + if (leftIsDigit) + { + return 1; + } + + if (rightIsDigit) + { + return -1; + } + + var leftOrder = CharOrder(leftChar); + var rightOrder = CharOrder(rightChar); + + var orderComparison = leftOrder.CompareTo(rightOrder); + if (orderComparison != 0) + { + return orderComparison; + } + + if (leftChar != rightChar) + { + return leftChar.CompareTo(rightChar); + } + + if (leftChar == '\0') + { + return 0; + } + + i++; + j++; + } + + return 0; + } + + private static bool IsAlphaNumeric(char value) + => char.IsLetterOrDigit(value); + + private static int CharOrder(char value) + { + if (value == '\0') + { + return 0; + } + + if (value == '~') + { + return -1; + } + + if (char.IsDigit(value)) + { + return 0; + } + + if (char.IsLetter(value)) + { + return value; + } + + return value + 256; + } +} diff --git a/src/StellaOps.Feedser.Merge/Comparers/Nevra.cs b/src/StellaOps.Feedser.Merge/Comparers/Nevra.cs new file mode 100644 index 00000000..4914a6ac --- /dev/null +++ b/src/StellaOps.Feedser.Merge/Comparers/Nevra.cs @@ -0,0 +1,264 @@ +namespace StellaOps.Feedser.Merge.Comparers; + +using System; +using StellaOps.Feedser.Normalization.Distro; + +public sealed class NevraComparer : IComparer, IComparer +{ + public static NevraComparer Instance { get; } = new(); + + private NevraComparer() + { + } + + public int Compare(string? x, string? y) + { + if (ReferenceEquals(x, y)) + { + return 0; + } + + if (x is null) + { + return -1; + } + + if (y is null) + { + return 1; + } + + var xParsed = Nevra.TryParse(x, out var xNevra); + var yParsed = Nevra.TryParse(y, out var yNevra); + + if (xParsed && yParsed) + { + return Compare(xNevra, yNevra); + } + + if (xParsed) + { + return 1; + } + + if (yParsed) + { + return -1; + } + + return string.Compare(x, y, StringComparison.Ordinal); + } + + public int Compare(Nevra? x, Nevra? y) + { + if (ReferenceEquals(x, y)) + { + return 0; + } + + if (x is null) + { + return -1; + } + + if (y is null) + { + return 1; + } + + var compare = string.Compare(x.Name, y.Name, StringComparison.Ordinal); + if (compare != 0) + { + return compare; + } + + compare = string.Compare(x.Architecture ?? string.Empty, y.Architecture ?? string.Empty, StringComparison.Ordinal); + if (compare != 0) + { + return compare; + } + + compare = x.Epoch.CompareTo(y.Epoch); + if (compare != 0) + { + return compare; + } + + compare = RpmVersionComparer.Compare(x.Version, y.Version); + if (compare != 0) + { + return compare; + } + + compare = RpmVersionComparer.Compare(x.Release, y.Release); + if (compare != 0) + { + return compare; + } + + return string.Compare(x.Original, y.Original, StringComparison.Ordinal); + } +} + +internal static class RpmVersionComparer +{ + public static int Compare(string? left, string? right) + { + left ??= string.Empty; + right ??= string.Empty; + + var i = 0; + var j = 0; + + while (true) + { + var leftHasTilde = SkipToNextSegment(left, ref i); + var rightHasTilde = SkipToNextSegment(right, ref j); + + if (leftHasTilde || rightHasTilde) + { + if (leftHasTilde && rightHasTilde) + { + continue; + } + + return leftHasTilde ? -1 : 1; + } + + var leftEnd = i >= left.Length; + var rightEnd = j >= right.Length; + if (leftEnd || rightEnd) + { + if (leftEnd && rightEnd) + { + return 0; + } + + return leftEnd ? -1 : 1; + } + + var leftDigit = char.IsDigit(left[i]); + var rightDigit = char.IsDigit(right[j]); + + if (leftDigit && !rightDigit) + { + return 1; + } + + if (!leftDigit && rightDigit) + { + return -1; + } + + int compare; + if (leftDigit) + { + compare = CompareNumericSegment(left, ref i, right, ref j); + } + else + { + compare = CompareAlphaSegment(left, ref i, right, ref j); + } + + if (compare != 0) + { + return compare; + } + } + } + + private static bool SkipToNextSegment(string value, ref int index) + { + var sawTilde = false; + while (index < value.Length) + { + var current = value[index]; + if (current == '~') + { + sawTilde = true; + index++; + break; + } + + if (char.IsLetterOrDigit(current)) + { + break; + } + + index++; + } + + return sawTilde; + } + + private static int CompareNumericSegment(string value, ref int index, string other, ref int otherIndex) + { + var start = index; + while (index < value.Length && char.IsDigit(value[index])) + { + index++; + } + + var otherStart = otherIndex; + while (otherIndex < other.Length && char.IsDigit(other[otherIndex])) + { + otherIndex++; + } + + var trimmedStart = start; + while (trimmedStart < index && value[trimmedStart] == '0') + { + trimmedStart++; + } + + var otherTrimmedStart = otherStart; + while (otherTrimmedStart < otherIndex && other[otherTrimmedStart] == '0') + { + otherTrimmedStart++; + } + + var length = index - trimmedStart; + var otherLength = otherIndex - otherTrimmedStart; + + if (length != otherLength) + { + return length.CompareTo(otherLength); + } + + var comparison = value.AsSpan(trimmedStart, length) + .CompareTo(other.AsSpan(otherTrimmedStart, otherLength), StringComparison.Ordinal); + if (comparison != 0) + { + return comparison; + } + + return 0; + } + + private static int CompareAlphaSegment(string value, ref int index, string other, ref int otherIndex) + { + var start = index; + while (index < value.Length && char.IsLetter(value[index])) + { + index++; + } + + var otherStart = otherIndex; + while (otherIndex < other.Length && char.IsLetter(other[otherIndex])) + { + otherIndex++; + } + + var length = index - start; + var otherLength = otherIndex - otherStart; + + var comparison = value.AsSpan(start, length) + .CompareTo(other.AsSpan(otherStart, otherLength), StringComparison.Ordinal); + if (comparison != 0) + { + return comparison; + } + + return 0; + } +} diff --git a/src/StellaOps.Feedser.Merge/Comparers/SemanticVersionRangeResolver.cs b/src/StellaOps.Feedser.Merge/Comparers/SemanticVersionRangeResolver.cs new file mode 100644 index 00000000..5b8c2fe7 --- /dev/null +++ b/src/StellaOps.Feedser.Merge/Comparers/SemanticVersionRangeResolver.cs @@ -0,0 +1,73 @@ +namespace StellaOps.Feedser.Merge.Comparers; + +using System.Diagnostics.CodeAnalysis; +using Semver; + +/// +/// Provides helpers to interpret introduced/fixed/lastAffected SemVer ranges and compare versions. +/// +public static class SemanticVersionRangeResolver +{ + public static bool TryParse(string? value, [NotNullWhen(true)] out SemVersion? result) + => SemVersion.TryParse(value, SemVersionStyles.Any, out result); + + public static SemVersion Parse(string value) + => SemVersion.Parse(value, SemVersionStyles.Any); + + /// + /// Resolves the effective start and end versions using introduced/fixed/lastAffected semantics. + /// + public static (SemVersion? introduced, SemVersion? exclusiveUpperBound, SemVersion? inclusiveUpperBound) ResolveWindows( + string? introduced, + string? fixedVersion, + string? lastAffected) + { + var introducedVersion = TryParse(introduced, out var parsedIntroduced) ? parsedIntroduced : null; + var fixedVersionParsed = TryParse(fixedVersion, out var parsedFixed) ? parsedFixed : null; + var lastAffectedVersion = TryParse(lastAffected, out var parsedLast) ? parsedLast : null; + + SemVersion? exclusiveUpper = null; + SemVersion? inclusiveUpper = null; + + if (fixedVersionParsed is not null) + { + exclusiveUpper = fixedVersionParsed; + } + else if (lastAffectedVersion is not null) + { + inclusiveUpper = lastAffectedVersion; + exclusiveUpper = NextPatch(lastAffectedVersion); + } + + return (introducedVersion, exclusiveUpper, inclusiveUpper); + } + + + public static int Compare(string? left, string? right) + { + var leftParsed = TryParse(left, out var leftSemver); + var rightParsed = TryParse(right, out var rightSemver); + + if (leftParsed && rightParsed) + { + return SemVersion.CompareSortOrder(leftSemver, rightSemver); + } + + if (leftParsed) + { + return 1; + } + + if (rightParsed) + { + return -1; + } + + return string.Compare(left, right, StringComparison.Ordinal); + } + + private static SemVersion NextPatch(SemVersion version) + { + return new SemVersion(version.Major, version.Minor, version.Patch + 1); + } +} diff --git a/src/StellaOps.Feedser.Merge/Identity/AdvisoryIdentityCluster.cs b/src/StellaOps.Feedser.Merge/Identity/AdvisoryIdentityCluster.cs new file mode 100644 index 00000000..97dd7dee --- /dev/null +++ b/src/StellaOps.Feedser.Merge/Identity/AdvisoryIdentityCluster.cs @@ -0,0 +1,56 @@ +using System; +using System.Collections.Generic; +using System.Collections.Immutable; +using System.Linq; +using StellaOps.Feedser.Models; + +namespace StellaOps.Feedser.Merge.Identity; + +/// +/// Represents a connected component of advisories that refer to the same vulnerability. +/// +public sealed class AdvisoryIdentityCluster +{ + public AdvisoryIdentityCluster(string advisoryKey, IEnumerable advisories, IEnumerable aliases) + { + AdvisoryKey = !string.IsNullOrWhiteSpace(advisoryKey) + ? advisoryKey.Trim() + : throw new ArgumentException("Canonical advisory key must be provided.", nameof(advisoryKey)); + + var advisoriesArray = (advisories ?? throw new ArgumentNullException(nameof(advisories))) + .Where(static advisory => advisory is not null) + .OrderBy(static advisory => advisory.AdvisoryKey, StringComparer.OrdinalIgnoreCase) + .ThenBy(static advisory => advisory.Provenance.Length) + .ThenBy(static advisory => advisory.Title, StringComparer.OrdinalIgnoreCase) + .ToImmutableArray(); + + if (advisoriesArray.IsDefaultOrEmpty) + { + throw new ArgumentException("At least one advisory is required for a cluster.", nameof(advisories)); + } + + var aliasArray = (aliases ?? throw new ArgumentNullException(nameof(aliases))) + .Where(static alias => alias is not null && !string.IsNullOrWhiteSpace(alias.Value)) + .GroupBy(static alias => alias.Value, StringComparer.OrdinalIgnoreCase) + .Select(static group => + { + var representative = group + .OrderBy(static entry => entry.Scheme ?? string.Empty, StringComparer.OrdinalIgnoreCase) + .ThenBy(static entry => entry.Value, StringComparer.OrdinalIgnoreCase) + .First(); + return representative; + }) + .OrderBy(static alias => alias.Scheme ?? string.Empty, StringComparer.OrdinalIgnoreCase) + .ThenBy(static alias => alias.Value, StringComparer.OrdinalIgnoreCase) + .ToImmutableArray(); + + Advisories = advisoriesArray; + Aliases = aliasArray; + } + + public string AdvisoryKey { get; } + + public ImmutableArray Advisories { get; } + + public ImmutableArray Aliases { get; } +} diff --git a/src/StellaOps.Feedser.Merge/Identity/AdvisoryIdentityResolver.cs b/src/StellaOps.Feedser.Merge/Identity/AdvisoryIdentityResolver.cs new file mode 100644 index 00000000..52c94599 --- /dev/null +++ b/src/StellaOps.Feedser.Merge/Identity/AdvisoryIdentityResolver.cs @@ -0,0 +1,303 @@ +using System; +using System.Collections.Generic; +using System.Collections.Immutable; +using System.Linq; +using System.Runtime.CompilerServices; +using StellaOps.Feedser.Models; + +namespace StellaOps.Feedser.Merge.Identity; + +/// +/// Builds an alias-driven identity graph that groups advisories referring to the same vulnerability. +/// +public sealed class AdvisoryIdentityResolver +{ + private static readonly string[] CanonicalAliasPriority = + { + AliasSchemes.Cve, + AliasSchemes.Rhsa, + AliasSchemes.Usn, + AliasSchemes.Dsa, + AliasSchemes.SuseSu, + AliasSchemes.Msrc, + AliasSchemes.CiscoSa, + AliasSchemes.OracleCpu, + AliasSchemes.Vmsa, + AliasSchemes.Apsb, + AliasSchemes.Apa, + AliasSchemes.AppleHt, + AliasSchemes.ChromiumPost, + AliasSchemes.Icsa, + AliasSchemes.Jvndb, + AliasSchemes.Jvn, + AliasSchemes.Bdu, + AliasSchemes.Vu, + AliasSchemes.Ghsa, + AliasSchemes.OsV, + }; + + /// + /// Groups the provided advisories into identity clusters using normalized aliases. + /// + public IReadOnlyList Resolve(IEnumerable advisories) + { + ArgumentNullException.ThrowIfNull(advisories); + + var materialized = advisories + .Where(static advisory => advisory is not null) + .Distinct() + .ToArray(); + + if (materialized.Length == 0) + { + return Array.Empty(); + } + + var aliasIndex = BuildAliasIndex(materialized); + var visited = new HashSet(); + var clusters = new List(); + + foreach (var advisory in materialized) + { + if (!visited.Add(advisory)) + { + continue; + } + + var component = TraverseComponent(advisory, visited, aliasIndex); + var key = DetermineCanonicalKey(component); + var aliases = component + .SelectMany(static entry => entry.Aliases) + .Select(static alias => new AliasIdentity(alias.Normalized, alias.Scheme)); + clusters.Add(new AdvisoryIdentityCluster(key, component.Select(static entry => entry.Advisory), aliases)); + } + + return clusters + .OrderBy(static cluster => cluster.AdvisoryKey, StringComparer.OrdinalIgnoreCase) + .ToImmutableArray(); + } + + private static Dictionary> BuildAliasIndex(IEnumerable advisories) + { + var index = new Dictionary>(StringComparer.OrdinalIgnoreCase); + + foreach (var advisory in advisories) + { + foreach (var alias in ExtractAliases(advisory)) + { + if (!index.TryGetValue(alias.Normalized, out var list)) + { + list = new List(); + index[alias.Normalized] = list; + } + + list.Add(new AdvisoryAliasEntry(advisory, alias.Normalized, alias.Scheme)); + } + } + + return index; + } + + private static IReadOnlyList TraverseComponent( + Advisory root, + HashSet visited, + Dictionary> aliasIndex) + { + var stack = new Stack(); + stack.Push(root); + + var bindings = new Dictionary(ReferenceEqualityComparer.Instance); + + while (stack.Count > 0) + { + var advisory = stack.Pop(); + + if (!bindings.TryGetValue(advisory, out var binding)) + { + binding = new AliasBinding(advisory); + bindings[advisory] = binding; + } + + foreach (var alias in ExtractAliases(advisory)) + { + binding.AddAlias(alias.Normalized, alias.Scheme); + + if (!aliasIndex.TryGetValue(alias.Normalized, out var neighbors)) + { + continue; + } + + foreach (var neighbor in neighbors.Select(static entry => entry.Advisory)) + { + if (visited.Add(neighbor)) + { + stack.Push(neighbor); + } + + if (!bindings.TryGetValue(neighbor, out var neighborBinding)) + { + neighborBinding = new AliasBinding(neighbor); + bindings[neighbor] = neighborBinding; + } + + neighborBinding.AddAlias(alias.Normalized, alias.Scheme); + } + } + } + + return bindings.Values + .OrderBy(static binding => binding.Advisory.AdvisoryKey, StringComparer.OrdinalIgnoreCase) + .ToImmutableArray(); + } + + private static string DetermineCanonicalKey(IReadOnlyList component) + { + var aliases = component + .SelectMany(static binding => binding.Aliases) + .Where(static alias => !string.IsNullOrWhiteSpace(alias.Normalized)) + .ToArray(); + + foreach (var scheme in CanonicalAliasPriority) + { + var candidate = aliases + .Where(alias => string.Equals(alias.Scheme, scheme, StringComparison.OrdinalIgnoreCase)) + .Select(static alias => alias.Normalized) + .OrderBy(static alias => alias, StringComparer.OrdinalIgnoreCase) + .FirstOrDefault(); + + if (candidate is not null) + { + return candidate; + } + } + + var fallbackAlias = aliases + .Select(static alias => alias.Normalized) + .OrderBy(static alias => alias, StringComparer.OrdinalIgnoreCase) + .FirstOrDefault(); + + if (!string.IsNullOrWhiteSpace(fallbackAlias)) + { + return fallbackAlias; + } + + var advisoryKey = component + .Select(static binding => binding.Advisory.AdvisoryKey) + .Where(static value => !string.IsNullOrWhiteSpace(value)) + .OrderBy(static value => value, StringComparer.OrdinalIgnoreCase) + .FirstOrDefault(); + + if (!string.IsNullOrWhiteSpace(advisoryKey)) + { + return advisoryKey.Trim(); + } + + throw new InvalidOperationException("Unable to determine canonical advisory key for cluster."); + } + + private static IEnumerable ExtractAliases(Advisory advisory) + { + var seen = new HashSet(StringComparer.OrdinalIgnoreCase); + + foreach (var candidate in EnumerateAliasCandidates(advisory)) + { + if (string.IsNullOrWhiteSpace(candidate)) + { + continue; + } + + var trimmed = candidate.Trim(); + if (!seen.Add(trimmed)) + { + continue; + } + + if (AliasSchemeRegistry.TryNormalize(trimmed, out var normalized, out var scheme) && + !string.IsNullOrWhiteSpace(normalized)) + { + yield return new AliasProjection(normalized.Trim(), string.IsNullOrWhiteSpace(scheme) ? null : scheme); + } + else if (!string.IsNullOrWhiteSpace(normalized)) + { + yield return new AliasProjection(normalized.Trim(), null); + } + } + } + + private static IEnumerable EnumerateAliasCandidates(Advisory advisory) + { + if (!string.IsNullOrWhiteSpace(advisory.AdvisoryKey)) + { + yield return advisory.AdvisoryKey; + } + + if (!advisory.Aliases.IsDefaultOrEmpty) + { + foreach (var alias in advisory.Aliases) + { + if (!string.IsNullOrWhiteSpace(alias)) + { + yield return alias; + } + } + } + } + + private readonly record struct AdvisoryAliasEntry(Advisory Advisory, string Normalized, string? Scheme); + + private readonly record struct AliasProjection(string Normalized, string? Scheme); + + private sealed class AliasBinding + { + private readonly HashSet _aliases = new(HashSetAliasComparer.Instance); + + public AliasBinding(Advisory advisory) + { + Advisory = advisory ?? throw new ArgumentNullException(nameof(advisory)); + } + + public Advisory Advisory { get; } + + public IReadOnlyCollection Aliases => _aliases; + + public void AddAlias(string normalized, string? scheme) + { + if (string.IsNullOrWhiteSpace(normalized)) + { + return; + } + + _aliases.Add(new AliasProjection(normalized.Trim(), scheme is null ? null : scheme.Trim())); + } + } + + private sealed class HashSetAliasComparer : IEqualityComparer + { + public static readonly HashSetAliasComparer Instance = new(); + + public bool Equals(AliasProjection x, AliasProjection y) + => string.Equals(x.Normalized, y.Normalized, StringComparison.OrdinalIgnoreCase) + && string.Equals(x.Scheme, y.Scheme, StringComparison.OrdinalIgnoreCase); + + public int GetHashCode(AliasProjection obj) + { + var hash = StringComparer.OrdinalIgnoreCase.GetHashCode(obj.Normalized); + if (!string.IsNullOrWhiteSpace(obj.Scheme)) + { + hash = HashCode.Combine(hash, StringComparer.OrdinalIgnoreCase.GetHashCode(obj.Scheme)); + } + + return hash; + } + } + + private sealed class ReferenceEqualityComparer : IEqualityComparer + where T : class + { + public static readonly ReferenceEqualityComparer Instance = new(); + + public bool Equals(T? x, T? y) => ReferenceEquals(x, y); + + public int GetHashCode(T obj) => RuntimeHelpers.GetHashCode(obj); + } +} diff --git a/src/StellaOps.Feedser.Merge/Identity/AliasIdentity.cs b/src/StellaOps.Feedser.Merge/Identity/AliasIdentity.cs new file mode 100644 index 00000000..193f7cc2 --- /dev/null +++ b/src/StellaOps.Feedser.Merge/Identity/AliasIdentity.cs @@ -0,0 +1,24 @@ +using System; + +namespace StellaOps.Feedser.Merge.Identity; + +/// +/// Normalized alias representation used within identity clusters. +/// +public sealed class AliasIdentity +{ + public AliasIdentity(string value, string? scheme) + { + if (string.IsNullOrWhiteSpace(value)) + { + throw new ArgumentException("Alias value must be provided.", nameof(value)); + } + + Value = value.Trim(); + Scheme = string.IsNullOrWhiteSpace(scheme) ? null : scheme.Trim(); + } + + public string Value { get; } + + public string? Scheme { get; } +} diff --git a/src/StellaOps.Feedser.Merge/Jobs/MergeJobKinds.cs b/src/StellaOps.Feedser.Merge/Jobs/MergeJobKinds.cs new file mode 100644 index 00000000..95cd7b3f --- /dev/null +++ b/src/StellaOps.Feedser.Merge/Jobs/MergeJobKinds.cs @@ -0,0 +1,6 @@ +namespace StellaOps.Feedser.Merge.Jobs; + +internal static class MergeJobKinds +{ + public const string Reconcile = "merge:reconcile"; +} diff --git a/src/StellaOps.Feedser.Merge/Jobs/MergeReconcileJob.cs b/src/StellaOps.Feedser.Merge/Jobs/MergeReconcileJob.cs new file mode 100644 index 00000000..5c98f117 --- /dev/null +++ b/src/StellaOps.Feedser.Merge/Jobs/MergeReconcileJob.cs @@ -0,0 +1,43 @@ +using System; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.Extensions.Logging; +using StellaOps.Feedser.Core.Jobs; +using StellaOps.Feedser.Merge.Services; + +namespace StellaOps.Feedser.Merge.Jobs; + +public sealed class MergeReconcileJob : IJob +{ + private readonly AdvisoryMergeService _mergeService; + private readonly ILogger _logger; + + public MergeReconcileJob(AdvisoryMergeService mergeService, ILogger logger) + { + _mergeService = mergeService ?? throw new ArgumentNullException(nameof(mergeService)); + _logger = logger ?? throw new ArgumentNullException(nameof(logger)); + } + + public async Task ExecuteAsync(JobExecutionContext context, CancellationToken cancellationToken) + { + if (!context.Parameters.TryGetValue("seed", out var seedValue) || seedValue is not string seed || string.IsNullOrWhiteSpace(seed)) + { + context.Logger.LogWarning("merge:reconcile job requires a non-empty 'seed' parameter."); + return; + } + + var result = await _mergeService.MergeAsync(seed, cancellationToken).ConfigureAwait(false); + if (result.Merged is null) + { + _logger.LogInformation("No advisories available to merge for alias component seeded by {Seed}", seed); + return; + } + + _logger.LogInformation( + "Merged alias component seeded by {Seed} into canonical {Canonical} using {Count} advisories; collisions={Collisions}", + seed, + result.CanonicalAdvisoryKey, + result.Inputs.Count, + result.Component.Collisions.Count); + } +} diff --git a/src/StellaOps.Feedser.Merge/MergeServiceCollectionExtensions.cs b/src/StellaOps.Feedser.Merge/MergeServiceCollectionExtensions.cs new file mode 100644 index 00000000..1896e6dc --- /dev/null +++ b/src/StellaOps.Feedser.Merge/MergeServiceCollectionExtensions.cs @@ -0,0 +1,43 @@ +using Microsoft.Extensions.Configuration; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.DependencyInjection.Extensions; +using Microsoft.Extensions.Logging; +using StellaOps.Feedser.Core; +using StellaOps.Feedser.Merge.Jobs; +using StellaOps.Feedser.Merge.Options; +using StellaOps.Feedser.Merge.Services; + +namespace StellaOps.Feedser.Merge; + +public static class MergeServiceCollectionExtensions +{ + public static IServiceCollection AddMergeModule(this IServiceCollection services, IConfiguration configuration) + { + ArgumentNullException.ThrowIfNull(services); + ArgumentNullException.ThrowIfNull(configuration); + + services.TryAddSingleton(); + services.TryAddSingleton(); + services.TryAddSingleton(); + services.TryAddSingleton(sp => + { + var options = configuration.GetSection("feedser:merge:precedence").Get(); + return options is null ? new AffectedPackagePrecedenceResolver() : new AffectedPackagePrecedenceResolver(options); + }); + + services.TryAddSingleton(sp => + { + var resolver = sp.GetRequiredService(); + var options = configuration.GetSection("feedser:merge:precedence").Get(); + var timeProvider = sp.GetRequiredService(); + var logger = sp.GetRequiredService>(); + return new AdvisoryPrecedenceMerger(resolver, options, timeProvider, logger); + }); + + services.TryAddSingleton(); + services.TryAddSingleton(); + services.AddTransient(); + + return services; + } +} diff --git a/src/StellaOps.Feedser.Merge/Options/AdvisoryPrecedenceDefaults.cs b/src/StellaOps.Feedser.Merge/Options/AdvisoryPrecedenceDefaults.cs new file mode 100644 index 00000000..5c22f4e3 --- /dev/null +++ b/src/StellaOps.Feedser.Merge/Options/AdvisoryPrecedenceDefaults.cs @@ -0,0 +1,96 @@ +using System; +using System.Collections.Generic; + +namespace StellaOps.Feedser.Merge.Options; + +/// +/// Provides the built-in precedence table used by the merge engine when no overrides are supplied. +/// +internal static class AdvisoryPrecedenceDefaults +{ + public static IReadOnlyDictionary Rankings { get; } = CreateDefaultTable(); + + private static IReadOnlyDictionary CreateDefaultTable() + { + var table = new Dictionary(StringComparer.OrdinalIgnoreCase); + + // 0 – distro PSIRTs/OVAL feeds (authoritative for OS packages). + Add(table, 0, + "redhat", + "ubuntu", + "distro-ubuntu", + "debian", + "distro-debian", + "suse", + "distro-suse"); + + // 1 – vendor PSIRTs (authoritative product advisories). + Add(table, 1, + "msrc", + "vndr-msrc", + "vndr-oracle", + "vndr_oracle", + "oracle", + "vndr-adobe", + "adobe", + "vndr-apple", + "apple", + "vndr-cisco", + "cisco", + "vmware", + "vndr-vmware", + "vndr_vmware", + "vndr-chromium", + "chromium", + "vendor"); + + // 2 – ecosystem registries (OSS package maintainers). + Add(table, 2, + "ghsa", + "osv", + "cve"); + + // 3 – regional CERT / ICS enrichment feeds. + Add(table, 3, + "jvn", + "acsc", + "cccs", + "cert-fr", + "certfr", + "cert-in", + "certin", + "cert-cc", + "certcc", + "certbund", + "cert-bund", + "ru-bdu", + "ru-nkcki", + "kisa", + "ics-cisa", + "ics-kaspersky"); + + // 4 – KEV / exploit catalogue annotations (flag only). + Add(table, 4, + "kev", + "cisa-kev"); + + // 5 – public registries (baseline data). + Add(table, 5, + "nvd"); + + return table; + } + + private static void Add(IDictionary table, int rank, params string[] sources) + { + foreach (var source in sources) + { + if (string.IsNullOrWhiteSpace(source)) + { + continue; + } + + table[source] = rank; + } + } +} diff --git a/src/StellaOps.Feedser.Merge/Options/AdvisoryPrecedenceOptions.cs b/src/StellaOps.Feedser.Merge/Options/AdvisoryPrecedenceOptions.cs new file mode 100644 index 00000000..bda12f2b --- /dev/null +++ b/src/StellaOps.Feedser.Merge/Options/AdvisoryPrecedenceOptions.cs @@ -0,0 +1,15 @@ +using System; +using System.Collections.Generic; + +namespace StellaOps.Feedser.Merge.Options; + +/// +/// Configurable precedence overrides for advisory sources. +/// +public sealed class AdvisoryPrecedenceOptions +{ + /// + /// Mapping of provenance source identifiers to precedence ranks. Lower numbers take precedence. + /// + public IDictionary Ranks { get; init; } = new Dictionary(StringComparer.OrdinalIgnoreCase); +} diff --git a/src/StellaOps.Feedser.Merge/Options/AdvisoryPrecedenceTable.cs b/src/StellaOps.Feedser.Merge/Options/AdvisoryPrecedenceTable.cs new file mode 100644 index 00000000..12bd9903 --- /dev/null +++ b/src/StellaOps.Feedser.Merge/Options/AdvisoryPrecedenceTable.cs @@ -0,0 +1,35 @@ +using System; +using System.Collections.Generic; + +namespace StellaOps.Feedser.Merge.Options; + +internal static class AdvisoryPrecedenceTable +{ + public static IReadOnlyDictionary Merge( + IReadOnlyDictionary defaults, + AdvisoryPrecedenceOptions? options) + { + if (defaults is null) + { + throw new ArgumentNullException(nameof(defaults)); + } + + if (options?.Ranks is null || options.Ranks.Count == 0) + { + return defaults; + } + + var merged = new Dictionary(defaults, StringComparer.OrdinalIgnoreCase); + foreach (var kvp in options.Ranks) + { + if (string.IsNullOrWhiteSpace(kvp.Key)) + { + continue; + } + + merged[kvp.Key.Trim()] = kvp.Value; + } + + return merged; + } +} diff --git a/src/StellaOps.Feedser.Merge/RANGE_PRIMITIVES_COORDINATION.md b/src/StellaOps.Feedser.Merge/RANGE_PRIMITIVES_COORDINATION.md new file mode 100644 index 00000000..e471bf7d --- /dev/null +++ b/src/StellaOps.Feedser.Merge/RANGE_PRIMITIVES_COORDINATION.md @@ -0,0 +1,44 @@ +# Range Primitive Coordination (Sprint 2) + +_Status date: 2025-10-11_ + +## Why this exists +- SemVer range outputs must follow the embedded rule guidance in `../FASTER_MODELING_AND_NORMALIZATION.md` (array of `{scheme,type,min/max/value,notes}`). +- Merge will rely on normalized rules plus existing `RangePrimitives` (SemVer/NEVRA/EVR) to dedupe ranges and compute deterministic hashes. +- Connector teams are mid-flight; this playbook restarts coordination so every feed delivers the normalized payload needed by the conflict resolver work in Sprint 3. + +## Upstream dependencies +- **Models** (`FEEDMODELS-SCHEMA-01-003`, `FEEDMODELS-SCHEMA-02-900`) – extends `RangePrimitives.SemVer` metadata and introduces `NormalizedVersionRule` arrays on affected packages. +- **Normalization** (`FEEDNORM-NORM-02-001`) – provides `SemVerRangeRuleBuilder` used by OSS connectors (GHSA/OSV/NVD) to emit canonical rule docs plus provenance notes. +- **Storage.Mongo** (`FEEDSTORAGE-DATA-02-001`) – dual-write/dual-read modifications for the new arrays; required before staging rollout. +- **Merge** (`FEEDMERGE-ENGINE-02-002`) – unions/dedupes normalized rules across sources once connectors publish them. + +Until these blocks land, connectors should stage changes behind a feature flag or fixture branch so we can flip on normalized writes in sync. + +## Connector adoption matrix +| Connector | Owner team | Current state (2025-10-11) | Required actions for SemVer guidance | Coordination notes | +|-----------|------------|-----------------------------|-------------------------------------|--------------------| +| Acsc | BE-Conn-ACSC | All tasks still TODO | Blocked on initial ingest work; align DTO design with normalized rule array before mapper lands. | Schedule pairing once `SemVerRangeRuleBuilder` API is published; ensure fixtures capture vendor/device taxonomy for provenance notes. | +| Cccs | BE-Conn-CCCS | All tasks still TODO | Same as Acsc; design DTOs with normalized rule hooks from day one. | Provide sample rule snippets in kickoff; share Mongo dual-write plan once storage flag is ready. | +| CertBund | BE-Conn-CERTBUND | All tasks still TODO | Ensure canonical mapper emits vendor range primitives plus normalized rules for product firmware. | Needs language/localisation guidance; coordinate with Localization WG for deterministic casing. | +| CertCc | BE-Conn-CERTCC | Fetch in progress, mapping TODO | Map VINCE vendor/product data into `RangePrimitives` with `certcc.vendor` extensions; build normalized SemVer ranges when version strings surface. | Follow up on 2025-10-14 to review VINCE payload examples and confirm builder requirements. | +| Cve | BE-Conn-CVE | Mapping/tests DONE (legacy SemVer) | Refactor `CveMapper` to call the shared builder and populate `NormalizedVersions` + provenance notes once models land. | Prepare MR behind `ENABLE_NORMALIZED_VERSIONS` flag; regression fixtures already cover version ranges—extend snapshots to cover rule arrays. | +| Ghsa | BE-Conn-GHSA | Mapping/tests DONE; normalized rule task TODO | Switch to `SemVerRangeRuleBuilder`, populate `NormalizedVersions`, and extend fixtures with rule/provenance fields. | Target code review window 2025-10-15; needs builder API from Normalization team by 2025-10-13. | +| Ics.Cisa | BE-Conn-ICS-CISA | All tasks TODO | When defining product schema, plan for SemVer or vendor version rules (many advisories use firmware revisions). | Gather sample advisories and confirm whether ranges are SemVer or vendor-specific so we can introduce scheme identifiers early. | +| Kisa | BE-Conn-KISA | All tasks TODO | Ensure DTO parsing captures version strings despite localisation; feed into normalized rule builder once ready. | Requires translation samples; request help from Localization WG before mapper implementation. | +| Ru.Bdu | BE-Conn-BDU | All tasks TODO | Map product releases into normalized rules; add provenance notes referencing BDU advisory identifiers. | Verify we have UTF-8 safe handling in builder; share sample sanitized inputs. | +| Ru.Nkcki | BE-Conn-Nkcki | All tasks TODO | Similar to BDU; capture vendor firmware/build numbers and map into normalized rules. | Coordinate with Localization WG for Cyrillic transliteration strategy. | +| Vndr.Apple | BE-Conn-Apple | Mapper/tests/telemetry marked DOING | Continue extending vendor range primitives (`apple.version`, `apple.build`) and adopt normalized rule arrays for OS build spans. | Request builder integration review on 2025-10-16; ensure fixtures cover multi-range tables and include provenance notes. | +| Vndr.Cisco | BE-Conn-Cisco | All tasks TODO | When parser lands, normalise IOS/ASA version strings into SemVer-style or vendor-specific ranges and supply normalized arrays. | Identify whether ranges require custom comparer (maybe `ios.semver` style); escalate to Models if new scheme required. | +| Vndr.Msrc | BE-Conn-MSRC | All tasks TODO | Canonical mapper must output product/build coverage as normalized rules (likely `msrc.patch` scheme) with provenance referencing KB IDs. | Sync with Models on adding scheme identifiers for MSRC packages; plan fixture coverage for monthly rollups. | + +## Immediate next steps +- Normalization team to share draft `SemVerRangeRuleBuilder` API by **2025-10-13** for review; Merge will circulate feedback within 24 hours. +- Connector owners to prepare fixture pull requests demonstrating sample normalized rule arrays (even if feature-flagged) by **2025-10-17**. +- Merge team will run a cross-connector review on **2025-10-18** to confirm consistent field usage and provenance tagging before enabling merge union logic. +- Schedule held for **2025-10-14 14:00 UTC** to review the CERT/CC staging VINCE advisory sample once `enableDetailMapping` is flipped; capture findings in `#feedser-merge` with snapshot diffs. + +## Tracking & follow-up +- Capture connector progress updates in stand-ups twice per week; link PRs/issues back to this document. +- When a connector is ready to emit normalized rules, update its module `TASKS.md` status and ping Merge in `#feedser-merge` with fixture diff screenshots. +- If new schemes or comparer logic is required (e.g., Cisco IOS), open a Models issue referencing `FEEDMODELS-SCHEMA-02-900` before implementing. diff --git a/src/StellaOps.Feedser.Merge/Services/AdvisoryMergeService.cs b/src/StellaOps.Feedser.Merge/Services/AdvisoryMergeService.cs new file mode 100644 index 00000000..148052ea --- /dev/null +++ b/src/StellaOps.Feedser.Merge/Services/AdvisoryMergeService.cs @@ -0,0 +1,291 @@ +using System; +using System.Collections.Generic; +using System.Collections.Immutable; +using System.Diagnostics.Metrics; +using System.Linq; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.Extensions.Logging; +using StellaOps.Feedser.Core; +using StellaOps.Feedser.Models; +using StellaOps.Feedser.Storage.Mongo.Advisories; +using StellaOps.Feedser.Storage.Mongo.Aliases; +using StellaOps.Feedser.Storage.Mongo.MergeEvents; + +namespace StellaOps.Feedser.Merge.Services; + +public sealed class AdvisoryMergeService +{ + private static readonly Meter MergeMeter = new("StellaOps.Feedser.Merge"); + private static readonly Counter AliasCollisionCounter = MergeMeter.CreateCounter( + "feedser.merge.identity_conflicts", + unit: "count", + description: "Number of alias collisions detected during merge."); + + private static readonly string[] PreferredAliasSchemes = + { + AliasSchemes.Cve, + AliasSchemes.Ghsa, + AliasSchemes.OsV, + AliasSchemes.Msrc, + }; + + private readonly AliasGraphResolver _aliasResolver; + private readonly IAdvisoryStore _advisoryStore; + private readonly AdvisoryPrecedenceMerger _precedenceMerger; + private readonly MergeEventWriter _mergeEventWriter; + private readonly CanonicalMerger _canonicalMerger; + private readonly ILogger _logger; + + public AdvisoryMergeService( + AliasGraphResolver aliasResolver, + IAdvisoryStore advisoryStore, + AdvisoryPrecedenceMerger precedenceMerger, + MergeEventWriter mergeEventWriter, + CanonicalMerger canonicalMerger, + ILogger logger) + { + _aliasResolver = aliasResolver ?? throw new ArgumentNullException(nameof(aliasResolver)); + _advisoryStore = advisoryStore ?? throw new ArgumentNullException(nameof(advisoryStore)); + _precedenceMerger = precedenceMerger ?? throw new ArgumentNullException(nameof(precedenceMerger)); + _mergeEventWriter = mergeEventWriter ?? throw new ArgumentNullException(nameof(mergeEventWriter)); + _canonicalMerger = canonicalMerger ?? throw new ArgumentNullException(nameof(canonicalMerger)); + _logger = logger ?? throw new ArgumentNullException(nameof(logger)); + } + + public async Task MergeAsync(string seedAdvisoryKey, CancellationToken cancellationToken) + { + ArgumentException.ThrowIfNullOrWhiteSpace(seedAdvisoryKey); + + var component = await _aliasResolver.BuildComponentAsync(seedAdvisoryKey, cancellationToken).ConfigureAwait(false); + var inputs = new List(); + + foreach (var advisoryKey in component.AdvisoryKeys) + { + cancellationToken.ThrowIfCancellationRequested(); + var advisory = await _advisoryStore.FindAsync(advisoryKey, cancellationToken).ConfigureAwait(false); + if (advisory is not null) + { + inputs.Add(advisory); + } + } + + if (inputs.Count == 0) + { + _logger.LogWarning("Alias component seeded by {Seed} contains no persisted advisories", seedAdvisoryKey); + return AdvisoryMergeResult.Empty(seedAdvisoryKey, component); + } + + var canonicalKey = SelectCanonicalKey(component) ?? seedAdvisoryKey; + var canonicalMerge = ApplyCanonicalMergeIfNeeded(canonicalKey, inputs); + var before = await _advisoryStore.FindAsync(canonicalKey, cancellationToken).ConfigureAwait(false); + var normalizedInputs = NormalizeInputs(inputs, canonicalKey).ToList(); + + Advisory? merged; + try + { + merged = _precedenceMerger.Merge(normalizedInputs); + } + catch (Exception ex) + { + _logger.LogError(ex, "Failed to merge alias component seeded by {Seed}", seedAdvisoryKey); + throw; + } + + if (component.Collisions.Count > 0) + { + foreach (var collision in component.Collisions) + { + var tags = new KeyValuePair[] + { + new("scheme", collision.Scheme ?? string.Empty), + new("alias_value", collision.Value ?? string.Empty), + new("advisory_count", collision.AdvisoryKeys.Count), + }; + + AliasCollisionCounter.Add(1, tags); + + _logger.LogInformation( + "Alias collision {Scheme}:{Value} involves advisories {Advisories}", + collision.Scheme, + collision.Value, + string.Join(", ", collision.AdvisoryKeys)); + } + } + + if (merged is not null) + { + await _advisoryStore.UpsertAsync(merged, cancellationToken).ConfigureAwait(false); + await _mergeEventWriter.AppendAsync( + canonicalKey, + before, + merged, + Array.Empty(), + ConvertFieldDecisions(canonicalMerge?.Decisions), + cancellationToken).ConfigureAwait(false); + } + + return new AdvisoryMergeResult(seedAdvisoryKey, canonicalKey, component, inputs, before, merged); + } + + private static IEnumerable NormalizeInputs(IEnumerable advisories, string canonicalKey) + { + foreach (var advisory in advisories) + { + yield return CloneWithKey(advisory, canonicalKey); + } + } + + private static Advisory CloneWithKey(Advisory source, string advisoryKey) + => new( + advisoryKey, + source.Title, + source.Summary, + source.Language, + source.Published, + source.Modified, + source.Severity, + source.ExploitKnown, + source.Aliases, + source.Credits, + source.References, + source.AffectedPackages, + source.CvssMetrics, + source.Provenance); + + private CanonicalMergeResult? ApplyCanonicalMergeIfNeeded(string canonicalKey, List inputs) + { + if (inputs.Count == 0) + { + return null; + } + + var ghsa = FindBySource(inputs, CanonicalSources.Ghsa); + var nvd = FindBySource(inputs, CanonicalSources.Nvd); + var osv = FindBySource(inputs, CanonicalSources.Osv); + + var participatingSources = 0; + if (ghsa is not null) + { + participatingSources++; + } + + if (nvd is not null) + { + participatingSources++; + } + + if (osv is not null) + { + participatingSources++; + } + + if (participatingSources < 2) + { + return null; + } + + var result = _canonicalMerger.Merge(canonicalKey, ghsa, nvd, osv); + + inputs.RemoveAll(advisory => MatchesCanonicalSource(advisory)); + inputs.Add(result.Advisory); + + return result; + } + + private static Advisory? FindBySource(IEnumerable advisories, string source) + => advisories.FirstOrDefault(advisory => advisory.Provenance.Any(provenance => + !string.Equals(provenance.Kind, "merge", StringComparison.OrdinalIgnoreCase) && + string.Equals(provenance.Source, source, StringComparison.OrdinalIgnoreCase))); + + private static bool MatchesCanonicalSource(Advisory advisory) + { + foreach (var provenance in advisory.Provenance) + { + if (string.Equals(provenance.Kind, "merge", StringComparison.OrdinalIgnoreCase)) + { + continue; + } + + if (string.Equals(provenance.Source, CanonicalSources.Ghsa, StringComparison.OrdinalIgnoreCase) || + string.Equals(provenance.Source, CanonicalSources.Nvd, StringComparison.OrdinalIgnoreCase) || + string.Equals(provenance.Source, CanonicalSources.Osv, StringComparison.OrdinalIgnoreCase)) + { + return true; + } + } + + return false; + } + + private static IReadOnlyList ConvertFieldDecisions(ImmutableArray? decisions) + { + if (decisions is null || decisions.Value.IsDefaultOrEmpty) + { + return Array.Empty(); + } + + var builder = ImmutableArray.CreateBuilder(decisions.Value.Length); + foreach (var decision in decisions.Value) + { + builder.Add(new MergeFieldDecision( + decision.Field, + decision.SelectedSource, + decision.DecisionReason, + decision.SelectedModified, + decision.ConsideredSources.ToArray())); + } + + return builder.ToImmutable(); + } + + private static class CanonicalSources + { + public const string Ghsa = "ghsa"; + public const string Nvd = "nvd"; + public const string Osv = "osv"; + } + + private static string? SelectCanonicalKey(AliasComponent component) + { + foreach (var scheme in PreferredAliasSchemes) + { + var alias = component.AliasMap.Values + .SelectMany(static aliases => aliases) + .FirstOrDefault(record => string.Equals(record.Scheme, scheme, StringComparison.OrdinalIgnoreCase)); + if (!string.IsNullOrWhiteSpace(alias?.Value)) + { + return alias.Value; + } + } + + if (component.AliasMap.TryGetValue(component.SeedAdvisoryKey, out var seedAliases)) + { + var primary = seedAliases.FirstOrDefault(record => string.Equals(record.Scheme, AliasStoreConstants.PrimaryScheme, StringComparison.OrdinalIgnoreCase)); + if (!string.IsNullOrWhiteSpace(primary?.Value)) + { + return primary.Value; + } + } + + var firstAlias = component.AliasMap.Values.SelectMany(static aliases => aliases).FirstOrDefault(); + if (!string.IsNullOrWhiteSpace(firstAlias?.Value)) + { + return firstAlias.Value; + } + + return component.SeedAdvisoryKey; + } +} + +public sealed record AdvisoryMergeResult( + string SeedAdvisoryKey, + string CanonicalAdvisoryKey, + AliasComponent Component, + IReadOnlyList Inputs, + Advisory? Previous, + Advisory? Merged) +{ + public static AdvisoryMergeResult Empty(string seed, AliasComponent component) + => new(seed, seed, component, Array.Empty(), null, null); +} diff --git a/src/StellaOps.Feedser.Merge/Services/AdvisoryPrecedenceMerger.cs b/src/StellaOps.Feedser.Merge/Services/AdvisoryPrecedenceMerger.cs new file mode 100644 index 00000000..bf676f6b --- /dev/null +++ b/src/StellaOps.Feedser.Merge/Services/AdvisoryPrecedenceMerger.cs @@ -0,0 +1,520 @@ +using System; +using System.Collections.Generic; +using System.Diagnostics.Metrics; +using System.Globalization; +using System.Linq; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Logging.Abstractions; +using StellaOps.Feedser.Merge.Options; +using StellaOps.Feedser.Models; + +namespace StellaOps.Feedser.Merge.Services; + +/// +/// Merges canonical advisories emitted by different sources into a single precedence-resolved advisory. +/// +public sealed class AdvisoryPrecedenceMerger +{ + private static readonly Meter MergeMeter = new("StellaOps.Feedser.Merge"); + private static readonly Counter MergeCounter = MergeMeter.CreateCounter( + "feedser.merge.operations", + unit: "count", + description: "Number of merge invocations executed by the precedence engine."); + + private static readonly Counter OverridesCounter = MergeMeter.CreateCounter( + "feedser.merge.overrides", + unit: "count", + description: "Number of times lower-precedence advisories were overridden by higher-precedence sources."); + + private static readonly Counter RangeOverrideCounter = MergeMeter.CreateCounter( + "feedser.merge.range_overrides", + unit: "count", + description: "Number of affected-package range overrides performed during precedence merge."); + + private static readonly Counter ConflictCounter = MergeMeter.CreateCounter( + "feedser.merge.conflicts", + unit: "count", + description: "Number of precedence conflicts detected (severity, rank ties, etc.)."); + + private static readonly Action OverrideLogged = LoggerMessage.Define( + LogLevel.Information, + new EventId(1000, "AdvisoryOverride"), + "Advisory precedence override {@Override}"); + + private static readonly Action RangeOverrideLogged = LoggerMessage.Define( + LogLevel.Information, + new EventId(1001, "PackageRangeOverride"), + "Affected package precedence override {@Override}"); + + private static readonly Action ConflictLogged = LoggerMessage.Define( + LogLevel.Information, + new EventId(1002, "PrecedenceConflict"), + "Precedence conflict {@Conflict}"); + + private readonly AffectedPackagePrecedenceResolver _packageResolver; + private readonly IReadOnlyDictionary _precedence; + private readonly int _fallbackRank; + private readonly System.TimeProvider _timeProvider; + private readonly ILogger _logger; + + public AdvisoryPrecedenceMerger() + : this(new AffectedPackagePrecedenceResolver(), TimeProvider.System) + { + } + + public AdvisoryPrecedenceMerger(AffectedPackagePrecedenceResolver packageResolver, System.TimeProvider? timeProvider = null) + : this(packageResolver, packageResolver?.Precedence ?? AdvisoryPrecedenceDefaults.Rankings, timeProvider ?? TimeProvider.System, NullLogger.Instance) + { + } + + public AdvisoryPrecedenceMerger( + AffectedPackagePrecedenceResolver packageResolver, + IReadOnlyDictionary precedence, + System.TimeProvider timeProvider) + : this(packageResolver, precedence, timeProvider, NullLogger.Instance) + { + } + + public AdvisoryPrecedenceMerger( + AffectedPackagePrecedenceResolver packageResolver, + AdvisoryPrecedenceOptions? options, + System.TimeProvider timeProvider, + ILogger? logger = null) + : this( + EnsureResolver(packageResolver, options, out var precedence), + precedence, + timeProvider, + logger) + { + } + + public AdvisoryPrecedenceMerger( + AffectedPackagePrecedenceResolver packageResolver, + IReadOnlyDictionary precedence, + System.TimeProvider timeProvider, + ILogger? logger) + { + _packageResolver = packageResolver ?? throw new ArgumentNullException(nameof(packageResolver)); + _precedence = precedence ?? throw new ArgumentNullException(nameof(precedence)); + _fallbackRank = _precedence.Count == 0 ? 10 : _precedence.Values.Max() + 1; + _timeProvider = timeProvider ?? TimeProvider.System; + _logger = logger ?? NullLogger.Instance; + } + + public Advisory Merge(IEnumerable advisories) + { + if (advisories is null) + { + throw new ArgumentNullException(nameof(advisories)); + } + + var list = advisories.Where(static a => a is not null).ToList(); + if (list.Count == 0) + { + throw new ArgumentException("At least one advisory is required for merge.", nameof(advisories)); + } + + var advisoryKey = list[0].AdvisoryKey; + if (list.Any(advisory => !string.Equals(advisory.AdvisoryKey, advisoryKey, StringComparison.Ordinal))) + { + throw new ArgumentException("All advisories must share the same advisory key.", nameof(advisories)); + } + + var ordered = list + .Select(advisory => new AdvisoryEntry(advisory, GetRank(advisory))) + .OrderBy(entry => entry.Rank) + .ThenByDescending(entry => entry.Advisory.Provenance.Length) + .ToArray(); + + MergeCounter.Add(1, new KeyValuePair("inputs", list.Count)); + + var primary = ordered[0].Advisory; + + var title = PickString(ordered, advisory => advisory.Title) ?? advisoryKey; + var summary = PickString(ordered, advisory => advisory.Summary); + var language = PickString(ordered, advisory => advisory.Language); + var severity = PickString(ordered, advisory => advisory.Severity); + + var aliases = ordered + .SelectMany(entry => entry.Advisory.Aliases) + .Where(static alias => !string.IsNullOrWhiteSpace(alias)) + .Distinct(StringComparer.OrdinalIgnoreCase) + .ToArray(); + + var credits = ordered + .SelectMany(entry => entry.Advisory.Credits) + .Distinct() + .ToArray(); + + var references = ordered + .SelectMany(entry => entry.Advisory.References) + .Distinct() + .ToArray(); + + var packageResult = _packageResolver.Merge(ordered.SelectMany(entry => entry.Advisory.AffectedPackages)); + var affectedPackages = packageResult.Packages; + var cvssMetrics = ordered + .SelectMany(entry => entry.Advisory.CvssMetrics) + .Distinct() + .ToArray(); + + var published = PickDateTime(ordered, static advisory => advisory.Published); + var modified = PickDateTime(ordered, static advisory => advisory.Modified) ?? published; + + var provenance = ordered + .SelectMany(entry => entry.Advisory.Provenance) + .Distinct() + .ToList(); + + var precedenceTrace = ordered + .SelectMany(entry => entry.Sources) + .Distinct(StringComparer.OrdinalIgnoreCase) + .OrderBy(static source => source, StringComparer.OrdinalIgnoreCase) + .ToArray(); + + var mergeProvenance = new AdvisoryProvenance( + source: "merge", + kind: "precedence", + value: string.Join("|", precedenceTrace), + recordedAt: _timeProvider.GetUtcNow()); + + provenance.Add(mergeProvenance); + + var exploitKnown = ordered.Any(entry => entry.Advisory.ExploitKnown); + + LogOverrides(advisoryKey, ordered); + LogPackageOverrides(advisoryKey, packageResult.Overrides); + RecordFieldConflicts(advisoryKey, ordered); + + return new Advisory( + advisoryKey, + title, + summary, + language, + published, + modified, + severity, + exploitKnown, + aliases, + credits, + references, + affectedPackages, + cvssMetrics, + provenance); + } + + private string? PickString(IEnumerable ordered, Func selector) + { + foreach (var entry in ordered) + { + var value = selector(entry.Advisory); + if (!string.IsNullOrWhiteSpace(value)) + { + return value.Trim(); + } + } + + return null; + } + + private DateTimeOffset? PickDateTime(IEnumerable ordered, Func selector) + { + foreach (var entry in ordered) + { + var value = selector(entry.Advisory); + if (value.HasValue) + { + return value.Value.ToUniversalTime(); + } + } + + return null; + } + + private int GetRank(Advisory advisory) + { + var best = _fallbackRank; + foreach (var provenance in advisory.Provenance) + { + if (string.IsNullOrWhiteSpace(provenance.Source)) + { + continue; + } + + if (_precedence.TryGetValue(provenance.Source, out var rank) && rank < best) + { + best = rank; + } + } + + return best; + } + + private void LogOverrides(string advisoryKey, IReadOnlyList ordered) + { + if (ordered.Count <= 1) + { + return; + } + + var primary = ordered[0]; + var primaryRank = primary.Rank; + + for (var i = 1; i < ordered.Count; i++) + { + var candidate = ordered[i]; + if (candidate.Rank <= primaryRank) + { + continue; + } + + var tags = new KeyValuePair[] + { + new("primary_source", FormatSourceLabel(primary.Sources)), + new("suppressed_source", FormatSourceLabel(candidate.Sources)), + new("primary_rank", primaryRank), + new("suppressed_rank", candidate.Rank), + }; + + OverridesCounter.Add(1, tags); + + var audit = new MergeOverrideAudit( + advisoryKey, + primary.Sources, + primaryRank, + candidate.Sources, + candidate.Rank, + primary.Advisory.Aliases.Length, + candidate.Advisory.Aliases.Length, + primary.Advisory.Provenance.Length, + candidate.Advisory.Provenance.Length); + + OverrideLogged(_logger, audit, null); + } + } + + private void LogPackageOverrides(string advisoryKey, IReadOnlyList overrides) + { + if (overrides.Count == 0) + { + return; + } + + foreach (var record in overrides) + { + var tags = new KeyValuePair[] + { + new("advisory_key", advisoryKey), + new("package_type", record.Type), + new("primary_source", FormatSourceLabel(record.PrimarySources)), + new("suppressed_source", FormatSourceLabel(record.SuppressedSources)), + new("primary_rank", record.PrimaryRank), + new("suppressed_rank", record.SuppressedRank), + new("primary_range_count", record.PrimaryRangeCount), + new("suppressed_range_count", record.SuppressedRangeCount), + }; + + RangeOverrideCounter.Add(1, tags); + + var audit = new PackageOverrideAudit( + advisoryKey, + record.Type, + record.Identifier, + record.Platform, + record.PrimaryRank, + record.SuppressedRank, + record.PrimarySources, + record.SuppressedSources, + record.PrimaryRangeCount, + record.SuppressedRangeCount); + + RangeOverrideLogged(_logger, audit, null); + } + } + + private void RecordFieldConflicts(string advisoryKey, IReadOnlyList ordered) + { + if (ordered.Count <= 1) + { + return; + } + + var primary = ordered[0]; + var primarySeverity = NormalizeSeverity(primary.Advisory.Severity); + + for (var i = 1; i < ordered.Count; i++) + { + var candidate = ordered[i]; + var candidateSeverity = NormalizeSeverity(candidate.Advisory.Severity); + + if (!string.IsNullOrEmpty(candidateSeverity)) + { + var reason = string.IsNullOrEmpty(primarySeverity) ? "primary_missing" : "mismatch"; + if (string.IsNullOrEmpty(primarySeverity) || !string.Equals(primarySeverity, candidateSeverity, StringComparison.OrdinalIgnoreCase)) + { + RecordConflict( + advisoryKey, + "severity", + reason, + primary, + candidate, + primarySeverity ?? "(none)", + candidateSeverity); + } + } + + if (candidate.Rank == primary.Rank) + { + RecordConflict( + advisoryKey, + "precedence_tie", + "equal_rank", + primary, + candidate, + primary.Rank.ToString(CultureInfo.InvariantCulture), + candidate.Rank.ToString(CultureInfo.InvariantCulture)); + } + } + } + + private void RecordConflict( + string advisoryKey, + string conflictType, + string reason, + AdvisoryEntry primary, + AdvisoryEntry suppressed, + string? primaryValue, + string? suppressedValue) + { + var tags = new KeyValuePair[] + { + new("type", conflictType), + new("reason", reason), + new("primary_source", FormatSourceLabel(primary.Sources)), + new("suppressed_source", FormatSourceLabel(suppressed.Sources)), + new("primary_rank", primary.Rank), + new("suppressed_rank", suppressed.Rank), + }; + + ConflictCounter.Add(1, tags); + + var audit = new MergeFieldConflictAudit( + advisoryKey, + conflictType, + reason, + primary.Sources, + primary.Rank, + suppressed.Sources, + suppressed.Rank, + primaryValue, + suppressedValue); + + ConflictLogged(_logger, audit, null); + } + + private readonly record struct AdvisoryEntry(Advisory Advisory, int Rank) + { + public IReadOnlyCollection Sources { get; } = Advisory.Provenance + .Select(static p => p.Source) + .Where(static source => !string.IsNullOrWhiteSpace(source)) + .Distinct(StringComparer.OrdinalIgnoreCase) + .ToArray(); + } + + private static string? NormalizeSeverity(string? severity) + => SeverityNormalization.Normalize(severity); + + private static AffectedPackagePrecedenceResolver EnsureResolver( + AffectedPackagePrecedenceResolver? resolver, + AdvisoryPrecedenceOptions? options, + out IReadOnlyDictionary precedence) + { + precedence = AdvisoryPrecedenceTable.Merge(AdvisoryPrecedenceDefaults.Rankings, options); + + if (resolver is null) + { + return new AffectedPackagePrecedenceResolver(precedence); + } + + if (DictionaryEquals(resolver.Precedence, precedence)) + { + return resolver; + } + + return new AffectedPackagePrecedenceResolver(precedence); + } + + private static bool DictionaryEquals( + IReadOnlyDictionary left, + IReadOnlyDictionary right) + { + if (ReferenceEquals(left, right)) + { + return true; + } + + if (left.Count != right.Count) + { + return false; + } + + foreach (var (key, value) in left) + { + if (!right.TryGetValue(key, out var other) || other != value) + { + return false; + } + } + + return true; + } + + private static string FormatSourceLabel(IReadOnlyCollection sources) + { + if (sources.Count == 0) + { + return "unknown"; + } + + if (sources.Count == 1) + { + return sources.First(); + } + + return string.Join('|', sources.OrderBy(static s => s, StringComparer.OrdinalIgnoreCase).Take(3)); + } + + private readonly record struct MergeOverrideAudit( + string AdvisoryKey, + IReadOnlyCollection PrimarySources, + int PrimaryRank, + IReadOnlyCollection SuppressedSources, + int SuppressedRank, + int PrimaryAliasCount, + int SuppressedAliasCount, + int PrimaryProvenanceCount, + int SuppressedProvenanceCount); + + private readonly record struct PackageOverrideAudit( + string AdvisoryKey, + string PackageType, + string Identifier, + string? Platform, + int PrimaryRank, + int SuppressedRank, + IReadOnlyCollection PrimarySources, + IReadOnlyCollection SuppressedSources, + int PrimaryRangeCount, + int SuppressedRangeCount); + + private readonly record struct MergeFieldConflictAudit( + string AdvisoryKey, + string ConflictType, + string Reason, + IReadOnlyCollection PrimarySources, + int PrimaryRank, + IReadOnlyCollection SuppressedSources, + int SuppressedRank, + string? PrimaryValue, + string? SuppressedValue); +} diff --git a/src/StellaOps.Feedser.Merge/Services/AffectedPackagePrecedenceResolver.cs b/src/StellaOps.Feedser.Merge/Services/AffectedPackagePrecedenceResolver.cs new file mode 100644 index 00000000..ea68ea16 --- /dev/null +++ b/src/StellaOps.Feedser.Merge/Services/AffectedPackagePrecedenceResolver.cs @@ -0,0 +1,163 @@ +using System; +using System.Collections.Generic; +using System.Collections.Immutable; +using System.Linq; +using StellaOps.Feedser.Merge.Options; +using StellaOps.Feedser.Models; + +namespace StellaOps.Feedser.Merge.Services; + +/// +/// Applies source precedence rules to affected package sets so authoritative distro ranges override generic registry data. +/// +public sealed class AffectedPackagePrecedenceResolver +{ + private readonly IReadOnlyDictionary _precedence; + private readonly int _fallbackRank; + + public AffectedPackagePrecedenceResolver() + : this(AdvisoryPrecedenceDefaults.Rankings) + { + } + + public AffectedPackagePrecedenceResolver(AdvisoryPrecedenceOptions? options) + : this(AdvisoryPrecedenceTable.Merge(AdvisoryPrecedenceDefaults.Rankings, options)) + { + } + + public AffectedPackagePrecedenceResolver(IReadOnlyDictionary precedence) + { + _precedence = precedence ?? throw new ArgumentNullException(nameof(precedence)); + _fallbackRank = precedence.Count == 0 ? 10 : precedence.Values.Max() + 1; + } + + public IReadOnlyDictionary Precedence => _precedence; + + public AffectedPackagePrecedenceResult Merge(IEnumerable packages) + { + ArgumentNullException.ThrowIfNull(packages); + + var grouped = packages + .Where(static pkg => pkg is not null) + .GroupBy(pkg => (pkg.Type, pkg.Identifier, pkg.Platform ?? string.Empty)); + + var resolved = new List(); + var overrides = new List(); + + foreach (var group in grouped) + { + var ordered = group + .Select(pkg => new PackageEntry(pkg, GetPrecedence(pkg))) + .OrderBy(static entry => entry.Rank) + .ThenByDescending(static entry => entry.Package.Provenance.Length) + .ThenByDescending(static entry => entry.Package.VersionRanges.Length) + .ToList(); + + var primary = ordered[0]; + var provenance = ordered + .SelectMany(static entry => entry.Package.Provenance) + .Where(static p => p is not null) + .Distinct() + .ToImmutableArray(); + + var statuses = ordered + .SelectMany(static entry => entry.Package.Statuses) + .Distinct(AffectedPackageStatusEqualityComparer.Instance) + .ToImmutableArray(); + + foreach (var candidate in ordered.Skip(1)) + { + if (candidate.Package.VersionRanges.Length == 0) + { + continue; + } + + overrides.Add(new AffectedPackageOverride( + primary.Package.Type, + primary.Package.Identifier, + string.IsNullOrWhiteSpace(primary.Package.Platform) ? null : primary.Package.Platform, + primary.Rank, + candidate.Rank, + ExtractSources(primary.Package), + ExtractSources(candidate.Package), + primary.Package.VersionRanges.Length, + candidate.Package.VersionRanges.Length)); + } + + var merged = new AffectedPackage( + primary.Type, + primary.Identifier, + string.IsNullOrWhiteSpace(primary.Platform) ? null : primary.Platform, + primary.Package.VersionRanges, + statuses, + provenance); + + resolved.Add(merged); + } + + var packagesResult = resolved + .OrderBy(static pkg => pkg.Type, StringComparer.Ordinal) + .ThenBy(static pkg => pkg.Identifier, StringComparer.Ordinal) + .ThenBy(static pkg => pkg.Platform, StringComparer.Ordinal) + .ToImmutableArray(); + + return new AffectedPackagePrecedenceResult(packagesResult, overrides.ToImmutableArray()); + } + + private int GetPrecedence(AffectedPackage package) + { + var bestRank = _fallbackRank; + foreach (var provenance in package.Provenance) + { + if (provenance is null || string.IsNullOrWhiteSpace(provenance.Source)) + { + continue; + } + + if (_precedence.TryGetValue(provenance.Source, out var rank) && rank < bestRank) + { + bestRank = rank; + } + } + + return bestRank; + } + + private static IReadOnlyList ExtractSources(AffectedPackage package) + { + if (package.Provenance.Length == 0) + { + return Array.Empty(); + } + + return package.Provenance + .Select(static p => p.Source) + .Where(static source => !string.IsNullOrWhiteSpace(source)) + .Distinct(StringComparer.OrdinalIgnoreCase) + .ToImmutableArray(); + } + + private readonly record struct PackageEntry(AffectedPackage Package, int Rank) + { + public string Type => Package.Type; + + public string Identifier => Package.Identifier; + + public string? Platform => string.IsNullOrWhiteSpace(Package.Platform) ? null : Package.Platform; + } +} + +public sealed record AffectedPackagePrecedenceResult( + IReadOnlyList Packages, + IReadOnlyList Overrides); + +public sealed record AffectedPackageOverride( + string Type, + string Identifier, + string? Platform, + int PrimaryRank, + int SuppressedRank, + IReadOnlyList PrimarySources, + IReadOnlyList SuppressedSources, + int PrimaryRangeCount, + int SuppressedRangeCount); diff --git a/src/StellaOps.Feedser.Merge/Services/AliasGraphResolver.cs b/src/StellaOps.Feedser.Merge/Services/AliasGraphResolver.cs new file mode 100644 index 00000000..5ff393f2 --- /dev/null +++ b/src/StellaOps.Feedser.Merge/Services/AliasGraphResolver.cs @@ -0,0 +1,139 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Threading; +using System.Threading.Tasks; +using StellaOps.Feedser.Storage.Mongo.Aliases; + +namespace StellaOps.Feedser.Merge.Services; + +public sealed class AliasGraphResolver +{ + private readonly IAliasStore _aliasStore; + + public AliasGraphResolver(IAliasStore aliasStore) + { + _aliasStore = aliasStore ?? throw new ArgumentNullException(nameof(aliasStore)); + } + + public async Task ResolveAsync(string advisoryKey, CancellationToken cancellationToken) + { + ArgumentException.ThrowIfNullOrEmpty(advisoryKey); + var aliases = await _aliasStore.GetByAdvisoryAsync(advisoryKey, cancellationToken).ConfigureAwait(false); + var collisions = new List(); + + foreach (var alias in aliases) + { + var candidates = await _aliasStore.GetByAliasAsync(alias.Scheme, alias.Value, cancellationToken).ConfigureAwait(false); + var advisoryKeys = candidates + .Select(static candidate => candidate.AdvisoryKey) + .Where(static key => !string.IsNullOrWhiteSpace(key)) + .Distinct(StringComparer.OrdinalIgnoreCase) + .ToArray(); + + if (advisoryKeys.Length <= 1) + { + continue; + } + + collisions.Add(new AliasCollision(alias.Scheme, alias.Value, advisoryKeys)); + } + + var unique = new Dictionary(StringComparer.Ordinal); + foreach (var collision in collisions) + { + var key = $"{collision.Scheme}\u0001{collision.Value}"; + if (!unique.ContainsKey(key)) + { + unique[key] = collision; + } + } + + var distinctCollisions = unique.Values.ToArray(); + + return new AliasIdentityResult(advisoryKey, aliases, distinctCollisions); + } + + public async Task BuildComponentAsync(string advisoryKey, CancellationToken cancellationToken) + { + ArgumentException.ThrowIfNullOrEmpty(advisoryKey); + + var visited = new HashSet(StringComparer.OrdinalIgnoreCase); + var queue = new Queue(); + var collisionMap = new Dictionary(StringComparer.Ordinal); + + var aliasCache = new Dictionary>(StringComparer.OrdinalIgnoreCase); + queue.Enqueue(advisoryKey); + + while (queue.Count > 0) + { + cancellationToken.ThrowIfCancellationRequested(); + var current = queue.Dequeue(); + if (!visited.Add(current)) + { + continue; + } + + var aliases = await GetAliasesAsync(current, cancellationToken, aliasCache).ConfigureAwait(false); + aliasCache[current] = aliases; + foreach (var alias in aliases) + { + var aliasRecords = await GetAdvisoriesForAliasAsync(alias.Scheme, alias.Value, cancellationToken).ConfigureAwait(false); + var advisoryKeys = aliasRecords + .Select(static record => record.AdvisoryKey) + .Where(static key => !string.IsNullOrWhiteSpace(key)) + .Distinct(StringComparer.OrdinalIgnoreCase) + .ToArray(); + + if (advisoryKeys.Length <= 1) + { + continue; + } + + foreach (var candidate in advisoryKeys) + { + if (!visited.Contains(candidate)) + { + queue.Enqueue(candidate); + } + } + + var collision = new AliasCollision(alias.Scheme, alias.Value, advisoryKeys); + var key = $"{collision.Scheme}\u0001{collision.Value}"; + collisionMap.TryAdd(key, collision); + } + } + + var aliasMap = new Dictionary>(aliasCache, StringComparer.OrdinalIgnoreCase); + return new AliasComponent(advisoryKey, visited.ToArray(), collisionMap.Values.ToArray(), aliasMap); + } + + private async Task> GetAliasesAsync( + string advisoryKey, + CancellationToken cancellationToken, + IDictionary> cache) + { + if (cache.TryGetValue(advisoryKey, out var cached)) + { + return cached; + } + + var aliases = await _aliasStore.GetByAdvisoryAsync(advisoryKey, cancellationToken).ConfigureAwait(false); + cache[advisoryKey] = aliases; + return aliases; + } + + private Task> GetAdvisoriesForAliasAsync( + string scheme, + string value, + CancellationToken cancellationToken) + => _aliasStore.GetByAliasAsync(scheme, value, cancellationToken); +} + +public sealed record AliasIdentityResult(string AdvisoryKey, IReadOnlyList Aliases, IReadOnlyList Collisions); + +public sealed record AliasComponent( + string SeedAdvisoryKey, + IReadOnlyList AdvisoryKeys, + IReadOnlyList Collisions, + IReadOnlyDictionary> AliasMap); diff --git a/src/StellaOps.Feedser.Merge/Services/CanonicalHashCalculator.cs b/src/StellaOps.Feedser.Merge/Services/CanonicalHashCalculator.cs new file mode 100644 index 00000000..7d6e9c4e --- /dev/null +++ b/src/StellaOps.Feedser.Merge/Services/CanonicalHashCalculator.cs @@ -0,0 +1,25 @@ +namespace StellaOps.Feedser.Merge.Services; + +using System.Security.Cryptography; +using System.Text; +using StellaOps.Feedser.Models; + +/// +/// Computes deterministic hashes over canonical advisory JSON payloads. +/// +public sealed class CanonicalHashCalculator +{ + private static readonly UTF8Encoding Utf8NoBom = new(false); + + public byte[] ComputeHash(Advisory? advisory) + { + if (advisory is null) + { + return Array.Empty(); + } + + var canonical = CanonicalJsonSerializer.Serialize(CanonicalJsonSerializer.Normalize(advisory)); + var payload = Utf8NoBom.GetBytes(canonical); + return SHA256.HashData(payload); + } +} diff --git a/src/StellaOps.Feedser.Merge/Services/MergeEventWriter.cs b/src/StellaOps.Feedser.Merge/Services/MergeEventWriter.cs new file mode 100644 index 00000000..6945e545 --- /dev/null +++ b/src/StellaOps.Feedser.Merge/Services/MergeEventWriter.cs @@ -0,0 +1,72 @@ +namespace StellaOps.Feedser.Merge.Services; + +using System.Security.Cryptography; +using System.Linq; +using Microsoft.Extensions.Logging; +using StellaOps.Feedser.Models; +using StellaOps.Feedser.Storage.Mongo.MergeEvents; + +/// +/// Persists merge events with canonical before/after hashes for auditability. +/// +public sealed class MergeEventWriter +{ + private readonly IMergeEventStore _mergeEventStore; + private readonly CanonicalHashCalculator _hashCalculator; + private readonly TimeProvider _timeProvider; + private readonly ILogger _logger; + + public MergeEventWriter( + IMergeEventStore mergeEventStore, + CanonicalHashCalculator hashCalculator, + TimeProvider timeProvider, + ILogger logger) + { + _mergeEventStore = mergeEventStore ?? throw new ArgumentNullException(nameof(mergeEventStore)); + _hashCalculator = hashCalculator ?? throw new ArgumentNullException(nameof(hashCalculator)); + _timeProvider = timeProvider ?? throw new ArgumentNullException(nameof(timeProvider)); + _logger = logger ?? throw new ArgumentNullException(nameof(logger)); + } + + public async Task AppendAsync( + string advisoryKey, + Advisory? before, + Advisory after, + IReadOnlyList inputDocumentIds, + IReadOnlyList? fieldDecisions, + CancellationToken cancellationToken) + { + ArgumentException.ThrowIfNullOrWhiteSpace(advisoryKey); + ArgumentNullException.ThrowIfNull(after); + + var beforeHash = _hashCalculator.ComputeHash(before); + var afterHash = _hashCalculator.ComputeHash(after); + var timestamp = _timeProvider.GetUtcNow(); + var documentIds = inputDocumentIds?.ToArray() ?? Array.Empty(); + + var record = new MergeEventRecord( + Guid.NewGuid(), + advisoryKey, + beforeHash, + afterHash, + timestamp, + documentIds, + fieldDecisions ?? Array.Empty()); + + if (!CryptographicOperations.FixedTimeEquals(beforeHash, afterHash)) + { + _logger.LogInformation( + "Merge event for {AdvisoryKey} changed hash {BeforeHash} -> {AfterHash}", + advisoryKey, + Convert.ToHexString(beforeHash), + Convert.ToHexString(afterHash)); + } + else + { + _logger.LogInformation("Merge event for {AdvisoryKey} recorded without hash change", advisoryKey); + } + + await _mergeEventStore.AppendAsync(record, cancellationToken).ConfigureAwait(false); + return record; + } +} diff --git a/src/StellaOps.Feedser.Merge/StellaOps.Feedser.Merge.csproj b/src/StellaOps.Feedser.Merge/StellaOps.Feedser.Merge.csproj new file mode 100644 index 00000000..d2bb7cd2 --- /dev/null +++ b/src/StellaOps.Feedser.Merge/StellaOps.Feedser.Merge.csproj @@ -0,0 +1,18 @@ + + + + + net10.0 + enable + enable + + + + + + + + + + + diff --git a/src/StellaOps.Feedser.Merge/TASKS.md b/src/StellaOps.Feedser.Merge/TASKS.md new file mode 100644 index 00000000..06479d3c --- /dev/null +++ b/src/StellaOps.Feedser.Merge/TASKS.md @@ -0,0 +1,18 @@ +# TASKS +| Task | Owner(s) | Depends on | Notes | +|---|---|---|---| +|Identity graph and alias resolver|BE-Merge|Models, Storage.Mongo|DONE – `AdvisoryIdentityResolver` builds alias-driven clusters with canonical key selection + unit coverage.| +|Precedence policy engine|BE-Merge|Architecture|**DONE** – precedence defaults enforced by `AdvisoryPrecedenceMerger`/`AdvisoryPrecedenceDefaults` with distro/PSIRT overriding registry feeds and CERT/KEV enrichers.| +|NEVRA comparer plus tests|BE-Merge (Distro WG)|Source.Distro fixtures|DONE – Added Nevra parser/comparer with tilde-aware rpm ordering and unit coverage.| +|Debian EVR comparer plus tests|BE-Merge (Distro WG)|Debian fixtures|DONE – DebianEvr comparer mirrors dpkg ordering with tilde/epoch handling and unit coverage.| +|SemVer range resolver plus tests|BE-Merge (OSS WG)|OSV/GHSA fixtures|DONE – SemanticVersionRangeResolver covers introduced/fixed/lastAffected semantics with SemVer ordering tests.| +|Canonical hash and merge_event writer|BE-Merge|Models, Storage.Mongo|DONE – Hash calculator + MergeEventWriter compute canonical SHA-256 digests and persist merge events.| +|Conflict detection and metrics|BE-Merge|Core|**DONE** – merge meters emit override/conflict counters and structured audits (`AdvisoryPrecedenceMerger`).| +|FEEDMERGE-ENGINE-04-001 GHSA/NVD/OSV conflict rules|BE-Merge|Core, Storage.Mongo|DONE – `AdvisoryMergeService` applies `CanonicalMerger` output before precedence merge, replacing source advisories with the canonical transcript. **Coordination:** connector fixture owners should surface canonical deltas to Merge QA before regression sign-off.| +|FEEDMERGE-ENGINE-04-002 Override metrics instrumentation|BE-Merge|Observability|DONE – merge events persist `MergeFieldDecision` records enabling analytics on precedence/freshness decisions. **Next:** hand off metrics schema to Ops for dashboard wiring.| +|FEEDMERGE-ENGINE-04-003 Reference & credit union pipeline|BE-Merge|Models|DONE – canonical merge preserves union semantics while respecting precedence, validated via updated credit union tests.| +|End-to-end determinism test|QA|Merge, key connectors|**DONE** – `MergePrecedenceIntegrationTests.MergePipeline_IsDeterministicAcrossRuns` guards determinism.| +|FEEDMERGE-QA-04-001 End-to-end conflict regression suite|QA|Merge|DONE – `AdvisoryMergeServiceTests.MergeAsync_AppliesCanonicalRulesAndPersistsDecisions` exercises GHSA/NVD/OSV conflict path and merge-event analytics. **Reminder:** QA to sync with connector teams once new fixture triples land.| +|Override audit logging|BE-Merge|Observability|DONE – override audits now emit structured logs plus bounded-tag metrics suitable for prod telemetry.| +|Configurable precedence table|BE-Merge|Architecture|DONE – precedence options bind via feedser:merge:precedence:ranks with docs/tests covering operator workflow.| +|Range primitives backlog|BE-Merge|Connector WGs|**DOING** – Coordinate remaining connectors (`Acsc`, `Cccs`, `CertBund`, `CertCc`, `Cve`, `Ghsa`, `Ics.Cisa`, `Kisa`, `Ru.Bdu`, `Ru.Nkcki`, `Vndr.Apple`, `Vndr.Cisco`, `Vndr.Msrc`) to emit canonical RangePrimitives with provenance tags; track progress/fixtures here.| diff --git a/src/StellaOps.Feedser.Models.Tests/AdvisoryProvenanceTests.cs b/src/StellaOps.Feedser.Models.Tests/AdvisoryProvenanceTests.cs new file mode 100644 index 00000000..b2f4e70a --- /dev/null +++ b/src/StellaOps.Feedser.Models.Tests/AdvisoryProvenanceTests.cs @@ -0,0 +1,49 @@ +using System; +using StellaOps.Feedser.Models; +using Xunit; + +namespace StellaOps.Feedser.Models.Tests; + +public sealed class AdvisoryProvenanceTests +{ + [Fact] + public void FieldMask_NormalizesAndDeduplicates() + { + var timestamp = DateTimeOffset.Parse("2025-01-01T00:00:00Z"); + var provenance = new AdvisoryProvenance( + source: "nvd", + kind: "map", + value: "CVE-2025-0001", + recordedAt: timestamp, + fieldMask: new[] { " AffectedPackages[] ", "affectedpackages[]", "references[]" }); + + Assert.Equal(timestamp, provenance.RecordedAt); + Assert.Collection( + provenance.FieldMask, + mask => Assert.Equal("affectedpackages[]", mask), + mask => Assert.Equal("references[]", mask)); + Assert.Null(provenance.DecisionReason); + } + + [Fact] + public void EmptyProvenance_ExposesEmptyFieldMask() + { + Assert.True(AdvisoryProvenance.Empty.FieldMask.IsEmpty); + Assert.Null(AdvisoryProvenance.Empty.DecisionReason); + } + + [Fact] + public void DecisionReason_IsTrimmed() + { + var timestamp = DateTimeOffset.Parse("2025-03-01T00:00:00Z"); + var provenance = new AdvisoryProvenance( + source: "merge", + kind: "precedence", + value: "summary", + recordedAt: timestamp, + fieldMask: new[] { ProvenanceFieldMasks.Advisory }, + decisionReason: " freshness_override "); + + Assert.Equal("freshness_override", provenance.DecisionReason); + } +} diff --git a/src/StellaOps.Feedser.Models.Tests/AdvisoryTests.cs b/src/StellaOps.Feedser.Models.Tests/AdvisoryTests.cs new file mode 100644 index 00000000..e8997e0b --- /dev/null +++ b/src/StellaOps.Feedser.Models.Tests/AdvisoryTests.cs @@ -0,0 +1,62 @@ +using System.Linq; +using StellaOps.Feedser.Models; + +namespace StellaOps.Feedser.Models.Tests; + +public sealed class AdvisoryTests +{ + [Fact] + public void CanonicalizesAliasesAndReferences() + { + var advisory = new Advisory( + advisoryKey: "TEST-123", + title: "Sample Advisory", + summary: " summary with spaces ", + language: "EN", + published: DateTimeOffset.Parse("2024-01-01T00:00:00Z"), + modified: DateTimeOffset.Parse("2024-01-02T00:00:00Z"), + severity: "CRITICAL", + exploitKnown: true, + aliases: new[] { " CVE-2024-0001", "GHSA-aaaa", "cve-2024-0001" }, + references: new[] + { + new AdvisoryReference("https://example.com/b", "patch", null, null, AdvisoryProvenance.Empty), + new AdvisoryReference("https://example.com/a", null, null, null, AdvisoryProvenance.Empty), + }, + affectedPackages: new[] + { + new AffectedPackage( + type: AffectedPackageTypes.SemVer, + identifier: "pkg:npm/sample", + platform: "node", + versionRanges: new[] + { + new AffectedVersionRange("semver", "1.0.0", "1.0.1", null, null, AdvisoryProvenance.Empty), + new AffectedVersionRange("semver", "1.0.0", "1.0.1", null, null, AdvisoryProvenance.Empty), + new AffectedVersionRange("semver", "0.9.0", null, "0.9.9", null, AdvisoryProvenance.Empty), + }, + statuses: Array.Empty(), + provenance: new[] + { + new AdvisoryProvenance("nvd", "map", "", DateTimeOffset.Parse("2024-01-01T00:00:00Z")), + new AdvisoryProvenance("vendor", "map", "", DateTimeOffset.Parse("2024-01-02T00:00:00Z")), + }) + }, + cvssMetrics: Array.Empty(), + provenance: new[] + { + new AdvisoryProvenance("nvd", "map", "", DateTimeOffset.Parse("2024-01-01T00:00:00Z")), + new AdvisoryProvenance("vendor", "map", "", DateTimeOffset.Parse("2024-01-02T00:00:00Z")), + }); + + Assert.Equal(new[] { "CVE-2024-0001", "GHSA-aaaa" }, advisory.Aliases); + Assert.Equal(new[] { "https://example.com/a", "https://example.com/b" }, advisory.References.Select(r => r.Url)); + Assert.Equal( + new[] + { + "semver|0.9.0||0.9.9|", + "semver|1.0.0|1.0.1||", + }, + advisory.AffectedPackages.Single().VersionRanges.Select(r => r.CreateDeterministicKey())); + } +} diff --git a/src/StellaOps.Feedser.Models.Tests/AffectedPackageStatusTests.cs b/src/StellaOps.Feedser.Models.Tests/AffectedPackageStatusTests.cs new file mode 100644 index 00000000..6bac3d85 --- /dev/null +++ b/src/StellaOps.Feedser.Models.Tests/AffectedPackageStatusTests.cs @@ -0,0 +1,75 @@ +using System; +using StellaOps.Feedser.Models; + +namespace StellaOps.Feedser.Models.Tests; + +public sealed class AffectedPackageStatusTests +{ + [Theory] + [InlineData("Known_Affected", AffectedPackageStatusCatalog.KnownAffected)] + [InlineData("KNOWN-NOT-AFFECTED", AffectedPackageStatusCatalog.KnownNotAffected)] + [InlineData("Under Investigation", AffectedPackageStatusCatalog.UnderInvestigation)] + [InlineData("Fixed", AffectedPackageStatusCatalog.Fixed)] + [InlineData("Known not vulnerable", AffectedPackageStatusCatalog.KnownNotAffected)] + [InlineData("Impacted", AffectedPackageStatusCatalog.Affected)] + [InlineData("Not Vulnerable", AffectedPackageStatusCatalog.NotAffected)] + [InlineData("Analysis in progress", AffectedPackageStatusCatalog.UnderInvestigation)] + [InlineData("Patch Available", AffectedPackageStatusCatalog.Fixed)] + [InlineData("Workaround available", AffectedPackageStatusCatalog.Mitigated)] + [InlineData("Does Not Apply", AffectedPackageStatusCatalog.NotApplicable)] + [InlineData("Awaiting fix", AffectedPackageStatusCatalog.Pending)] + [InlineData("TBD", AffectedPackageStatusCatalog.Unknown)] + public void Constructor_NormalizesStatus(string input, string expected) + { + var provenance = new AdvisoryProvenance("test", "status", "value", DateTimeOffset.UtcNow); + var status = new AffectedPackageStatus(input, provenance); + + Assert.Equal(expected, status.Status); + Assert.Equal(provenance, status.Provenance); + } + + [Fact] + public void Constructor_ThrowsForUnknownStatus() + { + var provenance = new AdvisoryProvenance("test", "status", "value", DateTimeOffset.UtcNow); + Assert.Throws(() => new AffectedPackageStatus("unsupported", provenance)); + } + + [Theory] + [InlineData("Not Impacted", AffectedPackageStatusCatalog.NotAffected)] + [InlineData("Resolved", AffectedPackageStatusCatalog.Fixed)] + [InlineData("Mitigation provided", AffectedPackageStatusCatalog.Mitigated)] + [InlineData("Out of scope", AffectedPackageStatusCatalog.NotApplicable)] + public void TryNormalize_ReturnsExpectedValue(string input, string expected) + { + Assert.True(AffectedPackageStatusCatalog.TryNormalize(input, out var normalized)); + Assert.Equal(expected, normalized); + } + + [Fact] + public void TryNormalize_ReturnsFalseForUnknown() + { + Assert.False(AffectedPackageStatusCatalog.TryNormalize("unsupported", out _)); + } + + [Fact] + public void Allowed_ReturnsCanonicalStatuses() + { + var expected = new[] + { + AffectedPackageStatusCatalog.KnownAffected, + AffectedPackageStatusCatalog.KnownNotAffected, + AffectedPackageStatusCatalog.UnderInvestigation, + AffectedPackageStatusCatalog.Fixed, + AffectedPackageStatusCatalog.FirstFixed, + AffectedPackageStatusCatalog.Mitigated, + AffectedPackageStatusCatalog.NotApplicable, + AffectedPackageStatusCatalog.Affected, + AffectedPackageStatusCatalog.NotAffected, + AffectedPackageStatusCatalog.Pending, + AffectedPackageStatusCatalog.Unknown, + }; + + Assert.Equal(expected, AffectedPackageStatusCatalog.Allowed); + } +} diff --git a/src/StellaOps.Feedser.Models.Tests/AliasSchemeRegistryTests.cs b/src/StellaOps.Feedser.Models.Tests/AliasSchemeRegistryTests.cs new file mode 100644 index 00000000..d197b70c --- /dev/null +++ b/src/StellaOps.Feedser.Models.Tests/AliasSchemeRegistryTests.cs @@ -0,0 +1,52 @@ +using StellaOps.Feedser.Models; + +namespace StellaOps.Feedser.Models.Tests; + +public sealed class AliasSchemeRegistryTests +{ + [Theory] + [InlineData("cve-2024-1234", AliasSchemes.Cve, "CVE-2024-1234")] + [InlineData("GHSA-xxxx-yyyy-zzzz", AliasSchemes.Ghsa, "GHSA-xxxx-yyyy-zzzz")] + [InlineData("osv-2023-15", AliasSchemes.OsV, "OSV-2023-15")] + [InlineData("jvndb-2023-123456", AliasSchemes.Jvndb, "JVNDB-2023-123456")] + [InlineData("vu#123456", AliasSchemes.Vu, "VU#123456")] + [InlineData("pkg:maven/org.example/app@1.0.0", AliasSchemes.Purl, "pkg:maven/org.example/app@1.0.0")] + [InlineData("cpe:/a:vendor:product:1.0", AliasSchemes.Cpe, "cpe:/a:vendor:product:1.0")] + public void TryNormalize_DetectsSchemeAndCanonicalizes(string input, string expectedScheme, string expectedAlias) + { + var success = AliasSchemeRegistry.TryNormalize(input, out var normalized, out var scheme); + + Assert.True(success); + Assert.Equal(expectedScheme, scheme); + Assert.Equal(expectedAlias, normalized); + } + + [Fact] + public void TryNormalize_ReturnsFalseForUnknownAlias() + { + var success = AliasSchemeRegistry.TryNormalize("custom-identifier", out var normalized, out var scheme); + + Assert.False(success); + Assert.Equal("custom-identifier", normalized); + Assert.Equal(string.Empty, scheme); + } + + [Fact] + public void Validation_NormalizesAliasWhenRecognized() + { + var result = Validation.TryNormalizeAlias(" rhsa-2024:0252 ", out var normalized); + + Assert.True(result); + Assert.NotNull(normalized); + Assert.Equal("RHSA-2024:0252", normalized); + } + + [Fact] + public void Validation_RejectsEmptyAlias() + { + var result = Validation.TryNormalizeAlias(" ", out var normalized); + + Assert.False(result); + Assert.Null(normalized); + } +} diff --git a/src/StellaOps.Feedser.Models.Tests/CanonicalExampleFactory.cs b/src/StellaOps.Feedser.Models.Tests/CanonicalExampleFactory.cs new file mode 100644 index 00000000..e308635a --- /dev/null +++ b/src/StellaOps.Feedser.Models.Tests/CanonicalExampleFactory.cs @@ -0,0 +1,195 @@ +using System.Collections.Generic; +using System.Globalization; +using StellaOps.Feedser.Models; + +namespace StellaOps.Feedser.Models.Tests; + +internal static class CanonicalExampleFactory +{ + public static IEnumerable<(string Name, Advisory Advisory)> GetExamples() + { + yield return ("nvd-basic", CreateNvdExample()); + yield return ("psirt-overlay", CreatePsirtOverlay()); + yield return ("ghsa-semver", CreateGhsaSemVer()); + yield return ("kev-flag", CreateKevFlag()); + } + + private static Advisory CreateNvdExample() + { + var provenance = Provenance("nvd", "map", "cve-2024-1234", "2024-08-01T12:00:00Z"); + return new Advisory( + advisoryKey: "CVE-2024-1234", + title: "Integer overflow in ExampleCMS", + summary: "An integer overflow in ExampleCMS allows remote attackers to escalate privileges.", + language: "en", + published: ParseDate("2024-07-15T00:00:00Z"), + modified: ParseDate("2024-07-16T10:35:00Z"), + severity: "high", + exploitKnown: false, + aliases: new[] { "CVE-2024-1234" }, + references: new[] + { + new AdvisoryReference( + "https://nvd.nist.gov/vuln/detail/CVE-2024-1234", + kind: "advisory", + sourceTag: "nvd", + summary: "NVD entry", + provenance: provenance), + new AdvisoryReference( + "https://example.org/security/CVE-2024-1234", + kind: "advisory", + sourceTag: "vendor", + summary: "Vendor bulletin", + provenance: Provenance("example", "fetch", "bulletin", "2024-07-14T15:00:00Z")), + }, + affectedPackages: new[] + { + new AffectedPackage( + type: AffectedPackageTypes.Cpe, + identifier: "cpe:/a:examplecms:examplecms:1.0", + platform: null, + versionRanges: new[] + { + new AffectedVersionRange("version", "1.0", "1.0.5", null, null, provenance), + }, + statuses: new[] + { + new AffectedPackageStatus("affected", provenance), + }, + provenance: new[] { provenance }), + }, + cvssMetrics: new[] + { + new CvssMetric("3.1", "CVSS:3.1/AV:N/AC:L/PR:N/UI:N/S:U/C:H/I:H/A:H", 9.8, "critical", provenance), + }, + provenance: new[] { provenance }); + } + + private static Advisory CreatePsirtOverlay() + { + var rhsaProv = Provenance("redhat", "map", "rhsa-2024:0252", "2024-05-11T09:00:00Z"); + var cveProv = Provenance("redhat", "enrich", "cve-2024-5678", "2024-05-11T09:05:00Z"); + return new Advisory( + advisoryKey: "RHSA-2024:0252", + title: "Important: kernel security update", + summary: "Updates the Red Hat Enterprise Linux kernel to address CVE-2024-5678.", + language: "en", + published: ParseDate("2024-05-10T19:28:00Z"), + modified: ParseDate("2024-05-11T08:15:00Z"), + severity: "critical", + exploitKnown: false, + aliases: new[] { "RHSA-2024:0252", "CVE-2024-5678" }, + references: new[] + { + new AdvisoryReference( + "https://access.redhat.com/errata/RHSA-2024:0252", + kind: "advisory", + sourceTag: "redhat", + summary: "Red Hat security advisory", + provenance: rhsaProv), + }, + affectedPackages: new[] + { + new AffectedPackage( + type: AffectedPackageTypes.Rpm, + identifier: "kernel-0:4.18.0-553.el8.x86_64", + platform: "rhel-8", + versionRanges: new[] + { + new AffectedVersionRange("nevra", "0:4.18.0-553.el8", null, null, null, rhsaProv), + }, + statuses: new[] + { + new AffectedPackageStatus("fixed", rhsaProv), + }, + provenance: new[] { rhsaProv, cveProv }), + }, + cvssMetrics: new[] + { + new CvssMetric("3.1", "CVSS:3.1/AV:L/AC:L/PR:H/UI:N/S:U/C:H/I:H/A:H", 6.7, "medium", rhsaProv), + }, + provenance: new[] { rhsaProv, cveProv }); + } + + private static Advisory CreateGhsaSemVer() + { + var provenance = Provenance("ghsa", "map", "ghsa-aaaa-bbbb-cccc", "2024-03-05T10:00:00Z"); + return new Advisory( + advisoryKey: "GHSA-aaaa-bbbb-cccc", + title: "Prototype pollution in widget.js", + summary: "A crafted payload can pollute Object.prototype leading to RCE.", + language: "en", + published: ParseDate("2024-03-04T00:00:00Z"), + modified: ParseDate("2024-03-04T12:00:00Z"), + severity: "high", + exploitKnown: false, + aliases: new[] { "GHSA-aaaa-bbbb-cccc", "CVE-2024-2222" }, + references: new[] + { + new AdvisoryReference( + "https://github.com/example/widget/security/advisories/GHSA-aaaa-bbbb-cccc", + kind: "advisory", + sourceTag: "ghsa", + summary: "GitHub Security Advisory", + provenance: provenance), + new AdvisoryReference( + "https://github.com/example/widget/commit/abcd1234", + kind: "patch", + sourceTag: "ghsa", + summary: "Patch commit", + provenance: provenance), + }, + affectedPackages: new[] + { + new AffectedPackage( + type: AffectedPackageTypes.SemVer, + identifier: "pkg:npm/example-widget", + platform: null, + versionRanges: new[] + { + new AffectedVersionRange("semver", null, "2.5.1", null, ">=0.0.0 <2.5.1", provenance), + new AffectedVersionRange("semver", "3.0.0", "3.2.4", null, null, provenance), + }, + statuses: Array.Empty(), + provenance: new[] { provenance }), + }, + cvssMetrics: new[] + { + new CvssMetric("3.1", "CVSS:3.1/AV:N/AC:L/PR:N/UI:R/S:U/C:H/I:H/A:H", 8.8, "high", provenance), + }, + provenance: new[] { provenance }); + } + + private static Advisory CreateKevFlag() + { + var provenance = Provenance("cisa-kev", "annotate", "kev", "2024-02-10T09:30:00Z"); + return new Advisory( + advisoryKey: "CVE-2023-9999", + title: "Remote code execution in LegacyServer", + summary: "Unauthenticated RCE due to unsafe deserialization.", + language: "en", + published: ParseDate("2023-11-20T00:00:00Z"), + modified: ParseDate("2024-02-09T16:22:00Z"), + severity: "critical", + exploitKnown: true, + aliases: new[] { "CVE-2023-9999" }, + references: new[] + { + new AdvisoryReference( + "https://www.cisa.gov/known-exploited-vulnerabilities-catalog", + kind: "kev", + sourceTag: "cisa", + summary: "CISA KEV entry", + provenance: provenance), + }, + affectedPackages: Array.Empty(), + cvssMetrics: Array.Empty(), + provenance: new[] { provenance }); + } + + private static AdvisoryProvenance Provenance(string source, string kind, string value, string recordedAt) + => new(source, kind, value, ParseDate(recordedAt)); + + private static DateTimeOffset ParseDate(string value) + => DateTimeOffset.Parse(value, CultureInfo.InvariantCulture, DateTimeStyles.AssumeUniversal).ToUniversalTime(); +} diff --git a/src/StellaOps.Feedser.Models.Tests/CanonicalExamplesTests.cs b/src/StellaOps.Feedser.Models.Tests/CanonicalExamplesTests.cs new file mode 100644 index 00000000..0b2864da --- /dev/null +++ b/src/StellaOps.Feedser.Models.Tests/CanonicalExamplesTests.cs @@ -0,0 +1,60 @@ +using System.Text; +using StellaOps.Feedser.Models; + +namespace StellaOps.Feedser.Models.Tests; + +public sealed class CanonicalExamplesTests +{ + private static readonly string FixtureRoot = Path.Combine(GetProjectRoot(), "Fixtures"); + private const string UpdateEnvVar = "UPDATE_GOLDENS"; + + [Trait("Category", "GoldenSnapshots")] + [Fact] + public void CanonicalExamplesMatchGoldenSnapshots() + { + Directory.CreateDirectory(FixtureRoot); + var envValue = Environment.GetEnvironmentVariable(UpdateEnvVar); + var updateGoldens = string.Equals(envValue, "1", StringComparison.OrdinalIgnoreCase); + var failures = new List(); + + foreach (var (name, advisory) in CanonicalExampleFactory.GetExamples()) + { + var snapshot = SnapshotSerializer.ToSnapshot(advisory).Replace("\r\n", "\n"); + var fixturePath = Path.Combine(FixtureRoot, $"{name}.json"); + + if (updateGoldens) + { + File.WriteAllText(fixturePath, snapshot); + continue; + } + + if (!File.Exists(fixturePath)) + { + failures.Add($"Missing golden fixture: {fixturePath}. Set {UpdateEnvVar}=1 to generate."); + continue; + } + + var expected = File.ReadAllText(fixturePath).Replace("\r\n", "\n"); + if (!string.Equals(expected, snapshot, StringComparison.Ordinal)) + { + var actualPath = Path.Combine(FixtureRoot, $"{name}.actual.json"); + File.WriteAllText(actualPath, snapshot); + failures.Add($"Fixture mismatch for {name}. Set {UpdateEnvVar}=1 to regenerate."); + } + } + + if (failures.Count > 0) + { + var builder = new StringBuilder(); + foreach (var failure in failures) + { + builder.AppendLine(failure); + } + + Assert.Fail(builder.ToString()); + } + } + + private static string GetProjectRoot() + => Path.GetFullPath(Path.Combine(AppContext.BaseDirectory, "..", "..", "..")); +} diff --git a/src/StellaOps.Feedser.Models.Tests/CanonicalJsonSerializerTests.cs b/src/StellaOps.Feedser.Models.Tests/CanonicalJsonSerializerTests.cs new file mode 100644 index 00000000..7fb361e2 --- /dev/null +++ b/src/StellaOps.Feedser.Models.Tests/CanonicalJsonSerializerTests.cs @@ -0,0 +1,152 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text.Json; +using StellaOps.Feedser.Models; + +namespace StellaOps.Feedser.Models.Tests; + +public sealed class CanonicalJsonSerializerTests +{ + [Fact] + public void SerializesPropertiesInDeterministicOrder() + { + var advisory = new Advisory( + advisoryKey: "TEST-321", + title: "Ordering", + summary: null, + language: null, + published: null, + modified: null, + severity: null, + exploitKnown: false, + aliases: new[] { "b", "a" }, + references: Array.Empty(), + affectedPackages: Array.Empty(), + cvssMetrics: Array.Empty(), + provenance: Array.Empty()); + + var json = CanonicalJsonSerializer.Serialize(advisory); + using var document = JsonDocument.Parse(json); + var propertyNames = document.RootElement.EnumerateObject().Select(p => p.Name).ToArray(); + + var sorted = propertyNames.OrderBy(name => name, StringComparer.Ordinal).ToArray(); + Assert.Equal(sorted, propertyNames); + } + + [Fact] + public void SnapshotSerializerProducesStableOutput() + { + var advisory = new Advisory( + advisoryKey: "TEST-999", + title: "Snapshot", + summary: "Example", + language: "EN", + published: DateTimeOffset.Parse("2024-06-01T00:00:00Z"), + modified: DateTimeOffset.Parse("2024-06-01T01:00:00Z"), + severity: "high", + exploitKnown: false, + aliases: new[] { "ALIAS-1" }, + references: new[] + { + new AdvisoryReference("https://example.com/a", "advisory", null, null, AdvisoryProvenance.Empty), + }, + affectedPackages: Array.Empty(), + cvssMetrics: Array.Empty(), + provenance: Array.Empty()); + + var snap1 = SnapshotSerializer.ToSnapshot(advisory); + var snap2 = SnapshotSerializer.ToSnapshot(advisory); + + Assert.Equal(snap1, snap2); + var normalized1 = snap1.Replace("\r\n", "\n"); + var normalized2 = snap2.Replace("\r\n", "\n"); + Assert.Equal(normalized1, normalized2); + } + + [Fact] + public void SerializesRangePrimitivesPayload() + { + var recordedAt = new DateTimeOffset(2025, 2, 1, 0, 0, 0, TimeSpan.Zero); + var provenance = new AdvisoryProvenance("connector-x", "map", "segment-1", recordedAt); + var primitives = new RangePrimitives( + new SemVerPrimitive( + Introduced: "2.0.0", + IntroducedInclusive: true, + Fixed: "2.3.4", + FixedInclusive: false, + LastAffected: "2.3.3", + LastAffectedInclusive: true, + ConstraintExpression: ">=2.0.0 <2.3.4"), + new NevraPrimitive( + Introduced: new NevraComponent("pkg", 0, "2.0.0", "1", "x86_64"), + Fixed: null, + LastAffected: new NevraComponent("pkg", 0, "2.3.3", "3", "x86_64")), + new EvrPrimitive( + Introduced: new EvrComponent(1, "2.0.0", "1"), + Fixed: new EvrComponent(1, "2.3.4", null), + LastAffected: null), + new Dictionary(StringComparer.Ordinal) + { + ["channel"] = "stable", + }); + + var range = new AffectedVersionRange( + rangeKind: "semver", + introducedVersion: "2.0.0", + fixedVersion: "2.3.4", + lastAffectedVersion: "2.3.3", + rangeExpression: ">=2.0.0 <2.3.4", + provenance, + primitives); + + var package = new AffectedPackage( + type: "semver", + identifier: "pkg@2.x", + platform: "linux", + versionRanges: new[] { range }, + statuses: Array.Empty(), + provenance: new[] { provenance }); + + var advisory = new Advisory( + advisoryKey: "TEST-PRIM", + title: "Range primitive serialization", + summary: null, + language: null, + published: recordedAt, + modified: recordedAt, + severity: null, + exploitKnown: false, + aliases: Array.Empty(), + references: Array.Empty(), + affectedPackages: new[] { package }, + cvssMetrics: Array.Empty(), + provenance: new[] { provenance }); + + var json = CanonicalJsonSerializer.Serialize(advisory); + using var document = JsonDocument.Parse(json); + var rangeElement = document.RootElement + .GetProperty("affectedPackages")[0] + .GetProperty("versionRanges")[0]; + + Assert.True(rangeElement.TryGetProperty("primitives", out var primitivesElement)); + + var semver = primitivesElement.GetProperty("semVer"); + Assert.Equal("2.0.0", semver.GetProperty("introduced").GetString()); + Assert.True(semver.GetProperty("introducedInclusive").GetBoolean()); + Assert.Equal("2.3.4", semver.GetProperty("fixed").GetString()); + Assert.False(semver.GetProperty("fixedInclusive").GetBoolean()); + Assert.Equal("2.3.3", semver.GetProperty("lastAffected").GetString()); + + var nevra = primitivesElement.GetProperty("nevra"); + Assert.Equal("pkg", nevra.GetProperty("introduced").GetProperty("name").GetString()); + Assert.Equal(0, nevra.GetProperty("introduced").GetProperty("epoch").GetInt32()); + + var evr = primitivesElement.GetProperty("evr"); + Assert.Equal(1, evr.GetProperty("introduced").GetProperty("epoch").GetInt32()); + Assert.Equal("2.3.4", evr.GetProperty("fixed").GetProperty("upstreamVersion").GetString()); + + var extensions = primitivesElement.GetProperty("vendorExtensions"); + Assert.Equal("stable", extensions.GetProperty("channel").GetString()); + } +} diff --git a/src/StellaOps.Feedser.Models.Tests/Fixtures/ghsa-semver.json b/src/StellaOps.Feedser.Models.Tests/Fixtures/ghsa-semver.json new file mode 100644 index 00000000..54c5c32a --- /dev/null +++ b/src/StellaOps.Feedser.Models.Tests/Fixtures/ghsa-semver.json @@ -0,0 +1,124 @@ +{ + "advisoryKey": "GHSA-aaaa-bbbb-cccc", + "affectedPackages": [ + { + "type": "semver", + "identifier": "pkg:npm/example-widget", + "platform": null, + "versionRanges": [ + { + "fixedVersion": "2.5.1", + "introducedVersion": null, + "lastAffectedVersion": null, + "primitives": null, + "provenance": { + "source": "ghsa", + "kind": "map", + "value": "ghsa-aaaa-bbbb-cccc", + "decisionReason": null, + "recordedAt": "2024-03-05T10:00:00+00:00", + "fieldMask": [] + }, + "rangeExpression": ">=0.0.0 <2.5.1", + "rangeKind": "semver" + }, + { + "fixedVersion": "3.2.4", + "introducedVersion": "3.0.0", + "lastAffectedVersion": null, + "primitives": null, + "provenance": { + "source": "ghsa", + "kind": "map", + "value": "ghsa-aaaa-bbbb-cccc", + "decisionReason": null, + "recordedAt": "2024-03-05T10:00:00+00:00", + "fieldMask": [] + }, + "rangeExpression": null, + "rangeKind": "semver" + } + ], + "normalizedVersions": [], + "statuses": [], + "provenance": [ + { + "source": "ghsa", + "kind": "map", + "value": "ghsa-aaaa-bbbb-cccc", + "decisionReason": null, + "recordedAt": "2024-03-05T10:00:00+00:00", + "fieldMask": [] + } + ] + } + ], + "aliases": [ + "CVE-2024-2222", + "GHSA-aaaa-bbbb-cccc" + ], + "credits": [], + "cvssMetrics": [ + { + "baseScore": 8.8, + "baseSeverity": "high", + "provenance": { + "source": "ghsa", + "kind": "map", + "value": "ghsa-aaaa-bbbb-cccc", + "decisionReason": null, + "recordedAt": "2024-03-05T10:00:00+00:00", + "fieldMask": [] + }, + "vector": "CVSS:3.1/AV:N/AC:L/PR:N/UI:R/S:U/C:H/I:H/A:H", + "version": "3.1" + } + ], + "exploitKnown": false, + "language": "en", + "modified": "2024-03-04T12:00:00+00:00", + "provenance": [ + { + "source": "ghsa", + "kind": "map", + "value": "ghsa-aaaa-bbbb-cccc", + "decisionReason": null, + "recordedAt": "2024-03-05T10:00:00+00:00", + "fieldMask": [] + } + ], + "published": "2024-03-04T00:00:00+00:00", + "references": [ + { + "kind": "patch", + "provenance": { + "source": "ghsa", + "kind": "map", + "value": "ghsa-aaaa-bbbb-cccc", + "decisionReason": null, + "recordedAt": "2024-03-05T10:00:00+00:00", + "fieldMask": [] + }, + "sourceTag": "ghsa", + "summary": "Patch commit", + "url": "https://github.com/example/widget/commit/abcd1234" + }, + { + "kind": "advisory", + "provenance": { + "source": "ghsa", + "kind": "map", + "value": "ghsa-aaaa-bbbb-cccc", + "decisionReason": null, + "recordedAt": "2024-03-05T10:00:00+00:00", + "fieldMask": [] + }, + "sourceTag": "ghsa", + "summary": "GitHub Security Advisory", + "url": "https://github.com/example/widget/security/advisories/GHSA-aaaa-bbbb-cccc" + } + ], + "severity": "high", + "summary": "A crafted payload can pollute Object.prototype leading to RCE.", + "title": "Prototype pollution in widget.js" +} \ No newline at end of file diff --git a/src/StellaOps.Feedser.Models.Tests/Fixtures/kev-flag.json b/src/StellaOps.Feedser.Models.Tests/Fixtures/kev-flag.json new file mode 100644 index 00000000..f0636285 --- /dev/null +++ b/src/StellaOps.Feedser.Models.Tests/Fixtures/kev-flag.json @@ -0,0 +1,42 @@ +{ + "advisoryKey": "CVE-2023-9999", + "affectedPackages": [], + "aliases": [ + "CVE-2023-9999" + ], + "credits": [], + "cvssMetrics": [], + "exploitKnown": true, + "language": "en", + "modified": "2024-02-09T16:22:00+00:00", + "provenance": [ + { + "source": "cisa-kev", + "kind": "annotate", + "value": "kev", + "decisionReason": null, + "recordedAt": "2024-02-10T09:30:00+00:00", + "fieldMask": [] + } + ], + "published": "2023-11-20T00:00:00+00:00", + "references": [ + { + "kind": "kev", + "provenance": { + "source": "cisa-kev", + "kind": "annotate", + "value": "kev", + "decisionReason": null, + "recordedAt": "2024-02-10T09:30:00+00:00", + "fieldMask": [] + }, + "sourceTag": "cisa", + "summary": "CISA KEV entry", + "url": "https://www.cisa.gov/known-exploited-vulnerabilities-catalog" + } + ], + "severity": "critical", + "summary": "Unauthenticated RCE due to unsafe deserialization.", + "title": "Remote code execution in LegacyServer" +} \ No newline at end of file diff --git a/src/StellaOps.Feedser.Models.Tests/Fixtures/nvd-basic.json b/src/StellaOps.Feedser.Models.Tests/Fixtures/nvd-basic.json new file mode 100644 index 00000000..bc8dc524 --- /dev/null +++ b/src/StellaOps.Feedser.Models.Tests/Fixtures/nvd-basic.json @@ -0,0 +1,119 @@ +{ + "advisoryKey": "CVE-2024-1234", + "affectedPackages": [ + { + "type": "cpe", + "identifier": "cpe:/a:examplecms:examplecms:1.0", + "platform": null, + "versionRanges": [ + { + "fixedVersion": "1.0.5", + "introducedVersion": "1.0", + "lastAffectedVersion": null, + "primitives": null, + "provenance": { + "source": "nvd", + "kind": "map", + "value": "cve-2024-1234", + "decisionReason": null, + "recordedAt": "2024-08-01T12:00:00+00:00", + "fieldMask": [] + }, + "rangeExpression": null, + "rangeKind": "version" + } + ], + "normalizedVersions": [], + "statuses": [ + { + "provenance": { + "source": "nvd", + "kind": "map", + "value": "cve-2024-1234", + "decisionReason": null, + "recordedAt": "2024-08-01T12:00:00+00:00", + "fieldMask": [] + }, + "status": "affected" + } + ], + "provenance": [ + { + "source": "nvd", + "kind": "map", + "value": "cve-2024-1234", + "decisionReason": null, + "recordedAt": "2024-08-01T12:00:00+00:00", + "fieldMask": [] + } + ] + } + ], + "aliases": [ + "CVE-2024-1234" + ], + "credits": [], + "cvssMetrics": [ + { + "baseScore": 9.8, + "baseSeverity": "critical", + "provenance": { + "source": "nvd", + "kind": "map", + "value": "cve-2024-1234", + "decisionReason": null, + "recordedAt": "2024-08-01T12:00:00+00:00", + "fieldMask": [] + }, + "vector": "CVSS:3.1/AV:N/AC:L/PR:N/UI:N/S:U/C:H/I:H/A:H", + "version": "3.1" + } + ], + "exploitKnown": false, + "language": "en", + "modified": "2024-07-16T10:35:00+00:00", + "provenance": [ + { + "source": "nvd", + "kind": "map", + "value": "cve-2024-1234", + "decisionReason": null, + "recordedAt": "2024-08-01T12:00:00+00:00", + "fieldMask": [] + } + ], + "published": "2024-07-15T00:00:00+00:00", + "references": [ + { + "kind": "advisory", + "provenance": { + "source": "example", + "kind": "fetch", + "value": "bulletin", + "decisionReason": null, + "recordedAt": "2024-07-14T15:00:00+00:00", + "fieldMask": [] + }, + "sourceTag": "vendor", + "summary": "Vendor bulletin", + "url": "https://example.org/security/CVE-2024-1234" + }, + { + "kind": "advisory", + "provenance": { + "source": "nvd", + "kind": "map", + "value": "cve-2024-1234", + "decisionReason": null, + "recordedAt": "2024-08-01T12:00:00+00:00", + "fieldMask": [] + }, + "sourceTag": "nvd", + "summary": "NVD entry", + "url": "https://nvd.nist.gov/vuln/detail/CVE-2024-1234" + } + ], + "severity": "high", + "summary": "An integer overflow in ExampleCMS allows remote attackers to escalate privileges.", + "title": "Integer overflow in ExampleCMS" +} \ No newline at end of file diff --git a/src/StellaOps.Feedser.Models.Tests/Fixtures/psirt-overlay.json b/src/StellaOps.Feedser.Models.Tests/Fixtures/psirt-overlay.json new file mode 100644 index 00000000..cc488094 --- /dev/null +++ b/src/StellaOps.Feedser.Models.Tests/Fixtures/psirt-overlay.json @@ -0,0 +1,122 @@ +{ + "advisoryKey": "RHSA-2024:0252", + "affectedPackages": [ + { + "type": "rpm", + "identifier": "kernel-0:4.18.0-553.el8.x86_64", + "platform": "rhel-8", + "versionRanges": [ + { + "fixedVersion": null, + "introducedVersion": "0:4.18.0-553.el8", + "lastAffectedVersion": null, + "primitives": null, + "provenance": { + "source": "redhat", + "kind": "map", + "value": "rhsa-2024:0252", + "decisionReason": null, + "recordedAt": "2024-05-11T09:00:00+00:00", + "fieldMask": [] + }, + "rangeExpression": null, + "rangeKind": "nevra" + } + ], + "normalizedVersions": [], + "statuses": [ + { + "provenance": { + "source": "redhat", + "kind": "map", + "value": "rhsa-2024:0252", + "decisionReason": null, + "recordedAt": "2024-05-11T09:00:00+00:00", + "fieldMask": [] + }, + "status": "fixed" + } + ], + "provenance": [ + { + "source": "redhat", + "kind": "enrich", + "value": "cve-2024-5678", + "decisionReason": null, + "recordedAt": "2024-05-11T09:05:00+00:00", + "fieldMask": [] + }, + { + "source": "redhat", + "kind": "map", + "value": "rhsa-2024:0252", + "decisionReason": null, + "recordedAt": "2024-05-11T09:00:00+00:00", + "fieldMask": [] + } + ] + } + ], + "aliases": [ + "CVE-2024-5678", + "RHSA-2024:0252" + ], + "credits": [], + "cvssMetrics": [ + { + "baseScore": 6.7, + "baseSeverity": "medium", + "provenance": { + "source": "redhat", + "kind": "map", + "value": "rhsa-2024:0252", + "decisionReason": null, + "recordedAt": "2024-05-11T09:00:00+00:00", + "fieldMask": [] + }, + "vector": "CVSS:3.1/AV:L/AC:L/PR:H/UI:N/S:U/C:H/I:H/A:H", + "version": "3.1" + } + ], + "exploitKnown": false, + "language": "en", + "modified": "2024-05-11T08:15:00+00:00", + "provenance": [ + { + "source": "redhat", + "kind": "enrich", + "value": "cve-2024-5678", + "decisionReason": null, + "recordedAt": "2024-05-11T09:05:00+00:00", + "fieldMask": [] + }, + { + "source": "redhat", + "kind": "map", + "value": "rhsa-2024:0252", + "decisionReason": null, + "recordedAt": "2024-05-11T09:00:00+00:00", + "fieldMask": [] + } + ], + "published": "2024-05-10T19:28:00+00:00", + "references": [ + { + "kind": "advisory", + "provenance": { + "source": "redhat", + "kind": "map", + "value": "rhsa-2024:0252", + "decisionReason": null, + "recordedAt": "2024-05-11T09:00:00+00:00", + "fieldMask": [] + }, + "sourceTag": "redhat", + "summary": "Red Hat security advisory", + "url": "https://access.redhat.com/errata/RHSA-2024:0252" + } + ], + "severity": "critical", + "summary": "Updates the Red Hat Enterprise Linux kernel to address CVE-2024-5678.", + "title": "Important: kernel security update" +} \ No newline at end of file diff --git a/src/StellaOps.Feedser.Models.Tests/NormalizedVersionRuleTests.cs b/src/StellaOps.Feedser.Models.Tests/NormalizedVersionRuleTests.cs new file mode 100644 index 00000000..7414eed6 --- /dev/null +++ b/src/StellaOps.Feedser.Models.Tests/NormalizedVersionRuleTests.cs @@ -0,0 +1,68 @@ +using System; +using System.Linq; +using StellaOps.Feedser.Models; +using Xunit; + +namespace StellaOps.Feedser.Models.Tests; + +public sealed class NormalizedVersionRuleTests +{ + [Fact] + public void NormalizedVersions_AreDeduplicatedAndOrdered() + { + var recordedAt = DateTimeOffset.Parse("2025-01-05T00:00:00Z"); + var provenance = new AdvisoryProvenance("ghsa", "map", "GHSA-abc", recordedAt); + var package = new AffectedPackage( + type: AffectedPackageTypes.SemVer, + identifier: "pkg:npm/example", + versionRanges: Array.Empty(), + normalizedVersions: new[] + { + new NormalizedVersionRule("SemVer", "Exact", value: "1.0.0 "), + new NormalizedVersionRule("semver", "range", min: "1.2.0", minInclusive: true, max: "2.0.0", maxInclusive: false, notes: "GHSA-abc"), + new NormalizedVersionRule("semver", "range", min: "1.2.0", minInclusive: true, max: "2.0.0", maxInclusive: false, notes: "GHSA-abc"), + new NormalizedVersionRule("semver", "gt", min: "0.9.0", minInclusive: false, notes: " originating nvd "), + }, + statuses: Array.Empty(), + provenance: new[] { provenance }); + + var normalized = package.NormalizedVersions.ToArray(); + Assert.Equal(3, normalized.Length); + + Assert.Collection( + normalized, + rule => + { + Assert.Equal("semver", rule.Scheme); + Assert.Equal("exact", rule.Type); + Assert.Equal("1.0.0", rule.Value); + Assert.Null(rule.Min); + Assert.Null(rule.Max); + }, + rule => + { + Assert.Equal("semver", rule.Scheme); + Assert.Equal("gt", rule.Type); + Assert.Equal("0.9.0", rule.Min); + Assert.False(rule.MinInclusive); + Assert.Equal("originating nvd", rule.Notes); + }, + rule => + { + Assert.Equal("semver", rule.Scheme); + Assert.Equal("range", rule.Type); + Assert.Equal("1.2.0", rule.Min); + Assert.True(rule.MinInclusive); + Assert.Equal("2.0.0", rule.Max); + Assert.False(rule.MaxInclusive); + Assert.Equal("GHSA-abc", rule.Notes); + }); + } + + [Fact] + public void NormalizedVersionRule_NormalizesTypeSeparators() + { + var rule = new NormalizedVersionRule("semver", "tie_breaker", value: "1.2.3"); + Assert.Equal("tie-breaker", rule.Type); + } +} diff --git a/src/StellaOps.Feedser.Models.Tests/OsvGhsaParityDiagnosticsTests.cs b/src/StellaOps.Feedser.Models.Tests/OsvGhsaParityDiagnosticsTests.cs new file mode 100644 index 00000000..38899e04 --- /dev/null +++ b/src/StellaOps.Feedser.Models.Tests/OsvGhsaParityDiagnosticsTests.cs @@ -0,0 +1,88 @@ +using System; +using System.Collections.Generic; +using System.Collections.Immutable; +using System.Diagnostics.Metrics; +using StellaOps.Feedser.Models; +using Xunit; + +namespace StellaOps.Feedser.Models.Tests; + +public sealed class OsvGhsaParityDiagnosticsTests +{ + [Fact] + public void RecordReport_EmitsTotalAndIssues() + { + var issues = ImmutableArray.Create( + new OsvGhsaParityIssue( + GhsaId: "GHSA-AAA", + IssueKind: "missing_osv", + Detail: "", + FieldMask: ImmutableArray.Create(ProvenanceFieldMasks.AffectedPackages)), + new OsvGhsaParityIssue( + GhsaId: "GHSA-BBB", + IssueKind: "severity_mismatch", + Detail: "", + FieldMask: ImmutableArray.Empty)); + var report = new OsvGhsaParityReport(2, issues); + + var measurements = new List<(string Instrument, long Value, IReadOnlyDictionary Tags)>(); + using var listener = CreateListener(measurements); + + OsvGhsaParityDiagnostics.RecordReport(report, "QA"); + + listener.Dispose(); + + Assert.Equal(3, measurements.Count); + + var total = Assert.Single(measurements, m => m.Instrument == "feedser.osv_ghsa.total"); + Assert.Equal(2, total.Value); + Assert.Equal("qa", total.Tags["dataset"]); + + var missing = Assert.Single(measurements, m => m.Tags.TryGetValue("issueKind", out var kind) && string.Equals((string)kind!, "missing_osv", StringComparison.Ordinal)); + Assert.Equal("affectedpackages[]", missing.Tags["fieldMask"]); + + var severity = Assert.Single(measurements, m => m.Tags.TryGetValue("issueKind", out var kind) && string.Equals((string)kind!, "severity_mismatch", StringComparison.Ordinal)); + Assert.Equal("none", severity.Tags["fieldMask"]); + } + + [Fact] + public void RecordReport_NoIssues_OnlyEmitsTotal() + { + var report = new OsvGhsaParityReport(0, ImmutableArray.Empty); + var measurements = new List<(string Instrument, long Value, IReadOnlyDictionary Tags)>(); + using var listener = CreateListener(measurements); + + OsvGhsaParityDiagnostics.RecordReport(report, ""); + + listener.Dispose(); + Assert.Empty(measurements); + } + + private static MeterListener CreateListener(List<(string Instrument, long Value, IReadOnlyDictionary Tags)> measurements) + { + var listener = new MeterListener + { + InstrumentPublished = (instrument, l) => + { + if (instrument.Meter.Name.StartsWith("StellaOps.Feedser.Models.OsvGhsaParity", StringComparison.Ordinal)) + { + l.EnableMeasurementEvents(instrument); + } + } + }; + + listener.SetMeasurementEventCallback((instrument, measurement, tags, state) => + { + var dict = new Dictionary(StringComparer.OrdinalIgnoreCase); + foreach (var tag in tags) + { + dict[tag.Key] = tag.Value; + } + + measurements.Add((instrument.Name, measurement, dict)); + }); + + listener.Start(); + return listener; + } +} diff --git a/src/StellaOps.Feedser.Models.Tests/OsvGhsaParityInspectorTests.cs b/src/StellaOps.Feedser.Models.Tests/OsvGhsaParityInspectorTests.cs new file mode 100644 index 00000000..b348d803 --- /dev/null +++ b/src/StellaOps.Feedser.Models.Tests/OsvGhsaParityInspectorTests.cs @@ -0,0 +1,148 @@ +using System; +using System.Collections.Generic; +using StellaOps.Feedser.Models; +using Xunit; + +namespace StellaOps.Feedser.Models.Tests; + +public sealed class OsvGhsaParityInspectorTests +{ + [Fact] + public void Compare_ReturnsNoIssues_WhenDatasetsMatch() + { + var ghsaId = "GHSA-1111"; + var osv = CreateOsvAdvisory(ghsaId, severity: "high", includeRanges: true); + var ghsa = CreateGhsaAdvisory(ghsaId, severity: "high", includeRanges: true); + + var report = OsvGhsaParityInspector.Compare(new[] { osv }, new[] { ghsa }); + + Assert.False(report.HasIssues); + Assert.Equal(1, report.TotalGhsaIds); + Assert.Empty(report.Issues); + } + + [Fact] + public void Compare_FlagsMissingOsvEntry() + { + var ghsaId = "GHSA-2222"; + var ghsa = CreateGhsaAdvisory(ghsaId, severity: "medium", includeRanges: true); + + var report = OsvGhsaParityInspector.Compare(Array.Empty(), new[] { ghsa }); + + var issue = Assert.Single(report.Issues); + Assert.Equal("missing_osv", issue.IssueKind); + Assert.Equal(ghsaId, issue.GhsaId); + Assert.Contains(ProvenanceFieldMasks.AffectedPackages, issue.FieldMask); + } + + [Fact] + public void Compare_FlagsMissingGhsaEntry() + { + var ghsaId = "GHSA-2424"; + var osv = CreateOsvAdvisory(ghsaId, severity: "medium", includeRanges: true); + + var report = OsvGhsaParityInspector.Compare(new[] { osv }, Array.Empty()); + + var issue = Assert.Single(report.Issues); + Assert.Equal("missing_ghsa", issue.IssueKind); + Assert.Equal(ghsaId, issue.GhsaId); + Assert.Contains(ProvenanceFieldMasks.AffectedPackages, issue.FieldMask); + } + + [Fact] + public void Compare_FlagsSeverityMismatch() + { + var ghsaId = "GHSA-3333"; + var osv = CreateOsvAdvisory(ghsaId, severity: "low", includeRanges: true); + var ghsa = CreateGhsaAdvisory(ghsaId, severity: "critical", includeRanges: true); + + var report = OsvGhsaParityInspector.Compare(new[] { osv }, new[] { ghsa }); + + var issue = Assert.Single(report.Issues, i => i.IssueKind == "severity_mismatch"); + Assert.Equal(ghsaId, issue.GhsaId); + Assert.Contains(ProvenanceFieldMasks.Advisory, issue.FieldMask); + } + + [Fact] + public void Compare_FlagsRangeMismatch() + { + var ghsaId = "GHSA-4444"; + var osv = CreateOsvAdvisory(ghsaId, severity: "high", includeRanges: false); + var ghsa = CreateGhsaAdvisory(ghsaId, severity: "high", includeRanges: true); + + var report = OsvGhsaParityInspector.Compare(new[] { osv }, new[] { ghsa }); + + var issue = Assert.Single(report.Issues, i => i.IssueKind == "range_mismatch"); + Assert.Equal(ghsaId, issue.GhsaId); + Assert.Contains(ProvenanceFieldMasks.VersionRanges, issue.FieldMask); + } + + private static Advisory CreateOsvAdvisory(string ghsaId, string? severity, bool includeRanges) + { + var timestamp = DateTimeOffset.UtcNow; + return new Advisory( + advisoryKey: $"osv-{ghsaId.ToLowerInvariant()}", + title: $"OSV {ghsaId}", + summary: null, + language: null, + published: timestamp, + modified: timestamp, + severity: severity, + exploitKnown: false, + aliases: new[] { ghsaId }, + references: Array.Empty(), + affectedPackages: includeRanges ? new[] { CreatePackage(timestamp, includeRanges) } : Array.Empty(), + cvssMetrics: Array.Empty(), + provenance: new[] + { + new AdvisoryProvenance("osv", "map", ghsaId, timestamp, new[] { ProvenanceFieldMasks.Advisory }) + }); + } + + private static Advisory CreateGhsaAdvisory(string ghsaId, string? severity, bool includeRanges) + { + var timestamp = DateTimeOffset.UtcNow; + return new Advisory( + advisoryKey: ghsaId.ToLowerInvariant(), + title: $"GHSA {ghsaId}", + summary: null, + language: null, + published: timestamp, + modified: timestamp, + severity: severity, + exploitKnown: false, + aliases: new[] { ghsaId }, + references: Array.Empty(), + affectedPackages: includeRanges ? new[] { CreatePackage(timestamp, includeRanges) } : Array.Empty(), + cvssMetrics: Array.Empty(), + provenance: new[] + { + new AdvisoryProvenance("ghsa", "map", ghsaId, timestamp, new[] { ProvenanceFieldMasks.Advisory }) + }); + } + + private static AffectedPackage CreatePackage(DateTimeOffset recordedAt, bool includeRanges) + { + var ranges = includeRanges + ? new[] + { + new AffectedVersionRange( + rangeKind: "semver", + introducedVersion: "1.0.0", + fixedVersion: "1.2.0", + lastAffectedVersion: null, + rangeExpression: null, + provenance: new AdvisoryProvenance("mapper", "range", "package@1", recordedAt, new[] { ProvenanceFieldMasks.VersionRanges }), + primitives: null) + } + : Array.Empty(); + + return new AffectedPackage( + type: "semver", + identifier: "pkg@1", + platform: null, + versionRanges: ranges, + statuses: Array.Empty(), + provenance: new[] { new AdvisoryProvenance("mapper", "package", "pkg@1", recordedAt, new[] { ProvenanceFieldMasks.AffectedPackages }) }); + } +} diff --git a/src/StellaOps.Feedser.Models.Tests/ProvenanceDiagnosticsTests.cs b/src/StellaOps.Feedser.Models.Tests/ProvenanceDiagnosticsTests.cs new file mode 100644 index 00000000..b410834b --- /dev/null +++ b/src/StellaOps.Feedser.Models.Tests/ProvenanceDiagnosticsTests.cs @@ -0,0 +1,180 @@ +using System; +using System.Collections.Generic; +using System.Diagnostics.Metrics; +using System.Linq; +using System.Reflection; +using Microsoft.Extensions.Logging.Abstractions; +using StellaOps.Feedser.Models; +using Xunit; + +namespace StellaOps.Feedser.Models.Tests; + +public sealed class ProvenanceDiagnosticsTests +{ + [Fact] + public void RecordMissing_AddsExpectedTagsAndDeduplicates() + { + ResetState(); + + var measurements = new List<(string Instrument, long Value, IReadOnlyDictionary Tags)>(); + using var listener = CreateListener(measurements); + + var baseline = DateTimeOffset.UtcNow; + ProvenanceDiagnostics.RecordMissing("source-A", "range:pkg", baseline, new[] { ProvenanceFieldMasks.VersionRanges }); + ProvenanceDiagnostics.RecordMissing("source-A", "range:pkg", baseline.AddMinutes(5), new[] { ProvenanceFieldMasks.VersionRanges }); + ProvenanceDiagnostics.RecordMissing("source-A", "reference:https://example", baseline.AddMinutes(10), new[] { ProvenanceFieldMasks.References }); + + listener.Dispose(); + + Assert.Equal(2, measurements.Count); + + var first = measurements[0]; + Assert.Equal(1, first.Value); + Assert.Equal("feedser.provenance.missing", first.Instrument); + Assert.Equal("source-A", first.Tags["source"]); + Assert.Equal("range:pkg", first.Tags["component"]); + Assert.Equal("range", first.Tags["category"]); + Assert.Equal("high", first.Tags["severity"]); + Assert.Equal(ProvenanceFieldMasks.VersionRanges, first.Tags["fieldMask"]); + + var second = measurements[1]; + Assert.Equal("feedser.provenance.missing", second.Instrument); + Assert.Equal("reference", second.Tags["category"]); + Assert.Equal("low", second.Tags["severity"]); + Assert.Equal(ProvenanceFieldMasks.References, second.Tags["fieldMask"]); + } + + [Fact] + public void ReportResumeWindow_ClearsTrackedEntries_WhenWindowBackfills() + { + ResetState(); + + var timestamp = DateTimeOffset.UtcNow; + ProvenanceDiagnostics.RecordMissing("source-B", "package:lib", timestamp); + + var (recorded, earliest, syncRoot) = GetInternalState(); + lock (syncRoot) + { + Assert.True(earliest.ContainsKey("source-B")); + Assert.Contains(recorded, entry => entry.StartsWith("source-B|", StringComparison.OrdinalIgnoreCase)); + } + + ProvenanceDiagnostics.ReportResumeWindow("source-B", timestamp.AddMinutes(-5), NullLogger.Instance); + + lock (syncRoot) + { + Assert.False(earliest.ContainsKey("source-B")); + Assert.DoesNotContain(recorded, entry => entry.StartsWith("source-B|", StringComparison.OrdinalIgnoreCase)); + } + } + + [Fact] + public void ReportResumeWindow_RetainsEntries_WhenWindowTooRecent() + { + ResetState(); + + var timestamp = DateTimeOffset.UtcNow; + ProvenanceDiagnostics.RecordMissing("source-C", "range:pkg", timestamp); + + ProvenanceDiagnostics.ReportResumeWindow("source-C", timestamp.AddMinutes(1), NullLogger.Instance); + + var (recorded, earliest, syncRoot) = GetInternalState(); + lock (syncRoot) + { + Assert.True(earliest.ContainsKey("source-C")); + Assert.Contains(recorded, entry => entry.StartsWith("source-C|", StringComparison.OrdinalIgnoreCase)); + } + } + + [Fact] + public void RecordRangePrimitive_EmitsCoverageMetric() + { + var range = new AffectedVersionRange( + rangeKind: "evr", + introducedVersion: "1:1.1.1n-0+deb11u2", + fixedVersion: null, + lastAffectedVersion: null, + rangeExpression: null, + provenance: new AdvisoryProvenance("source-D", "range", "pkg", DateTimeOffset.UtcNow), + primitives: new RangePrimitives( + SemVer: null, + Nevra: null, + Evr: new EvrPrimitive( + new EvrComponent(1, "1.1.1n", "0+deb11u2"), + null, + null), + VendorExtensions: new Dictionary { ["debian.release"] = "bullseye" })); + + var measurements = new List<(string Instrument, long Value, IReadOnlyDictionary Tags)>(); + using var listener = CreateListener(measurements, "feedser.range.primitives"); + + ProvenanceDiagnostics.RecordRangePrimitive("source-D", range); + + listener.Dispose(); + + var record = Assert.Single(measurements); + Assert.Equal("feedser.range.primitives", record.Instrument); + Assert.Equal(1, record.Value); + Assert.Equal("source-D", record.Tags["source"]); + Assert.Equal("evr", record.Tags["rangeKind"]); + Assert.Equal("evr", record.Tags["primitiveKinds"]); + Assert.Equal("true", record.Tags["hasVendorExtensions"]); + } + + private static MeterListener CreateListener( + List<(string Instrument, long Value, IReadOnlyDictionary Tags)> measurements, + params string[] instrumentNames) + { + var allowed = instrumentNames is { Length: > 0 } ? instrumentNames : new[] { "feedser.provenance.missing" }; + var allowedSet = new HashSet(allowed, StringComparer.OrdinalIgnoreCase); + + var listener = new MeterListener + { + InstrumentPublished = (instrument, l) => + { + if (instrument.Meter.Name == "StellaOps.Feedser.Models.Provenance" && allowedSet.Contains(instrument.Name)) + { + l.EnableMeasurementEvents(instrument); + } + } + }; + + listener.SetMeasurementEventCallback((instrument, measurement, tags, state) => + { + var dict = new Dictionary(StringComparer.OrdinalIgnoreCase); + foreach (var tag in tags) + { + dict[tag.Key] = tag.Value; + } + + measurements.Add((instrument.Name, measurement, dict)); + }); + + listener.Start(); + return listener; + } + + private static void ResetState() + { + var (_, _, syncRoot) = GetInternalState(); + lock (syncRoot) + { + var (recorded, earliest, _) = GetInternalState(); + recorded.Clear(); + earliest.Clear(); + } + } + + private static (HashSet Recorded, Dictionary Earliest, object SyncRoot) GetInternalState() + { + var type = typeof(ProvenanceDiagnostics); + var recordedField = type.GetField("RecordedComponents", BindingFlags.NonPublic | BindingFlags.Static) ?? throw new InvalidOperationException("RecordedComponents not found"); + var earliestField = type.GetField("EarliestMissing", BindingFlags.NonPublic | BindingFlags.Static) ?? throw new InvalidOperationException("EarliestMissing not found"); + var syncField = type.GetField("SyncRoot", BindingFlags.NonPublic | BindingFlags.Static) ?? throw new InvalidOperationException("SyncRoot not found"); + + var recorded = (HashSet)recordedField.GetValue(null)!; + var earliest = (Dictionary)earliestField.GetValue(null)!; + var sync = syncField.GetValue(null)!; + return (recorded, earliest, sync); + } +} diff --git a/src/StellaOps.Feedser.Models.Tests/RangePrimitivesTests.cs b/src/StellaOps.Feedser.Models.Tests/RangePrimitivesTests.cs new file mode 100644 index 00000000..82e55ea2 --- /dev/null +++ b/src/StellaOps.Feedser.Models.Tests/RangePrimitivesTests.cs @@ -0,0 +1,41 @@ +using System.Collections.Generic; +using StellaOps.Feedser.Models; +using Xunit; + +namespace StellaOps.Feedser.Models.Tests; + +public sealed class RangePrimitivesTests +{ + [Fact] + public void GetCoverageTag_ReturnsSpecificKinds() + { + var primitives = new RangePrimitives( + new SemVerPrimitive("1.0.0", true, "1.2.0", false, null, false, null), + new NevraPrimitive(null, null, null), + null, + null); + + Assert.Equal("nevra+semver", primitives.GetCoverageTag()); + } + + [Fact] + public void GetCoverageTag_ReturnsVendorWhenOnlyExtensions() + { + var primitives = new RangePrimitives( + null, + null, + null, + new Dictionary { ["vendor.status"] = "beta" }); + + Assert.True(primitives.HasVendorExtensions); + Assert.Equal("vendor", primitives.GetCoverageTag()); + } + + [Fact] + public void GetCoverageTag_ReturnsNoneWhenEmpty() + { + var primitives = new RangePrimitives(null, null, null, null); + Assert.False(primitives.HasVendorExtensions); + Assert.Equal("none", primitives.GetCoverageTag()); + } +} diff --git a/src/StellaOps.Feedser.Models.Tests/SemVerPrimitiveTests.cs b/src/StellaOps.Feedser.Models.Tests/SemVerPrimitiveTests.cs new file mode 100644 index 00000000..0f0e3760 --- /dev/null +++ b/src/StellaOps.Feedser.Models.Tests/SemVerPrimitiveTests.cs @@ -0,0 +1,132 @@ +using StellaOps.Feedser.Models; +using Xunit; + +namespace StellaOps.Feedser.Models.Tests; + +public sealed class SemVerPrimitiveTests +{ + [Theory] + [InlineData("1.0.0", true, "2.0.0", false, null, false, null, null, SemVerPrimitiveStyles.Range)] + [InlineData("1.0.0", true, null, false, null, false, null, null, SemVerPrimitiveStyles.GreaterThanOrEqual)] + [InlineData("1.0.0", false, null, false, null, false, null, null, SemVerPrimitiveStyles.GreaterThan)] + [InlineData(null, true, "2.0.0", false, null, false, null, null, SemVerPrimitiveStyles.LessThan)] + [InlineData(null, true, "2.0.0", true, null, false, null, null, SemVerPrimitiveStyles.LessThanOrEqual)] + [InlineData(null, true, null, false, "2.0.0", true, null, null, SemVerPrimitiveStyles.LessThanOrEqual)] + [InlineData(null, true, null, false, "2.0.0", false, null, null, SemVerPrimitiveStyles.LessThan)] + [InlineData(null, true, null, false, null, false, null, "1.5.0", SemVerPrimitiveStyles.Exact)] + public void StyleReflectsSemantics( + string? introduced, + bool introducedInclusive, + string? fixedVersion, + bool fixedInclusive, + string? lastAffected, + bool lastAffectedInclusive, + string? constraintExpression, + string? exactValue, + string expectedStyle) + { + var primitive = new SemVerPrimitive( + introduced, + introducedInclusive, + fixedVersion, + fixedInclusive, + lastAffected, + lastAffectedInclusive, + constraintExpression, + exactValue); + + Assert.Equal(expectedStyle, primitive.Style); + } + + [Fact] + public void EqualityIncludesExactValue() + { + var baseline = new SemVerPrimitive( + Introduced: null, + IntroducedInclusive: true, + Fixed: null, + FixedInclusive: false, + LastAffected: null, + LastAffectedInclusive: false, + ConstraintExpression: null); + + var variant = baseline with { ExactValue = "1.2.3" }; + + Assert.NotEqual(baseline, variant); + Assert.Equal(SemVerPrimitiveStyles.Exact, variant.Style); + Assert.Equal(SemVerPrimitiveStyles.Range, baseline.Style); + } + + [Fact] + public void ToNormalizedVersionRule_MapsRangeBounds() + { + var primitive = new SemVerPrimitive( + Introduced: "1.0.0", + IntroducedInclusive: true, + Fixed: "2.0.0", + FixedInclusive: false, + LastAffected: null, + LastAffectedInclusive: false, + ConstraintExpression: ">=1.0.0 <2.0.0"); + + var rule = primitive.ToNormalizedVersionRule(); + + Assert.NotNull(rule); + Assert.Equal(NormalizedVersionSchemes.SemVer, rule!.Scheme); + Assert.Equal(NormalizedVersionRuleTypes.Range, rule.Type); + Assert.Equal("1.0.0", rule.Min); + Assert.True(rule.MinInclusive); + Assert.Equal("2.0.0", rule.Max); + Assert.False(rule.MaxInclusive); + Assert.Null(rule.Value); + Assert.Equal(">=1.0.0 <2.0.0", rule.Notes); + } + + [Fact] + public void ToNormalizedVersionRule_ExactUsesExactValue() + { + var primitive = new SemVerPrimitive( + Introduced: null, + IntroducedInclusive: true, + Fixed: null, + FixedInclusive: false, + LastAffected: null, + LastAffectedInclusive: false, + ConstraintExpression: null, + ExactValue: "3.1.4"); + + var rule = primitive.ToNormalizedVersionRule("from-ghsa"); + + Assert.NotNull(rule); + Assert.Equal(NormalizedVersionSchemes.SemVer, rule!.Scheme); + Assert.Equal(NormalizedVersionRuleTypes.Exact, rule.Type); + Assert.Null(rule.Min); + Assert.Null(rule.Max); + Assert.Equal("3.1.4", rule.Value); + Assert.Equal("from-ghsa", rule.Notes); + } + + [Fact] + public void ToNormalizedVersionRule_GreaterThanMapsMinimum() + { + var primitive = new SemVerPrimitive( + Introduced: "1.5.0", + IntroducedInclusive: false, + Fixed: null, + FixedInclusive: false, + LastAffected: null, + LastAffectedInclusive: false, + ConstraintExpression: null); + + var rule = primitive.ToNormalizedVersionRule(); + + Assert.NotNull(rule); + Assert.Equal(NormalizedVersionSchemes.SemVer, rule!.Scheme); + Assert.Equal(NormalizedVersionRuleTypes.GreaterThan, rule.Type); + Assert.Equal("1.5.0", rule.Min); + Assert.False(rule.MinInclusive); + Assert.Null(rule.Max); + Assert.Null(rule.Value); + Assert.Null(rule.Notes); + } +} diff --git a/src/StellaOps.Feedser.Models.Tests/SerializationDeterminismTests.cs b/src/StellaOps.Feedser.Models.Tests/SerializationDeterminismTests.cs new file mode 100644 index 00000000..0f04755f --- /dev/null +++ b/src/StellaOps.Feedser.Models.Tests/SerializationDeterminismTests.cs @@ -0,0 +1,68 @@ +using System; +using System.Collections.Generic; +using System.Globalization; +using System.Linq; +using StellaOps.Feedser.Models; +using Xunit; + +namespace StellaOps.Feedser.Models.Tests; + +public sealed class SerializationDeterminismTests +{ + private static readonly string[] Cultures = + { + "en-US", + "fr-FR", + "tr-TR", + "ja-JP", + "ar-SA" + }; + + [Fact] + public void CanonicalSerializer_ProducesStableJsonAcrossCultures() + { + var examples = CanonicalExampleFactory.GetExamples().ToArray(); + var baseline = SerializeUnderCulture(CultureInfo.InvariantCulture, examples); + + foreach (var cultureName in Cultures) + { + var culture = CultureInfo.GetCultureInfo(cultureName); + var serialized = SerializeUnderCulture(culture, examples); + + Assert.Equal(baseline.Count, serialized.Count); + for (var i = 0; i < baseline.Count; i++) + { + Assert.Equal(baseline[i].Compact, serialized[i].Compact); + Assert.Equal(baseline[i].Indented, serialized[i].Indented); + } + } + } + + private static List<(string Name, string Compact, string Indented)> SerializeUnderCulture( + CultureInfo culture, + IReadOnlyList<(string Name, Advisory Advisory)> examples) + { + var originalCulture = CultureInfo.CurrentCulture; + var originalUiCulture = CultureInfo.CurrentUICulture; + try + { + CultureInfo.CurrentCulture = culture; + CultureInfo.CurrentUICulture = culture; + + var results = new List<(string Name, string Compact, string Indented)>(examples.Count); + foreach (var (name, advisory) in examples) + { + var compact = CanonicalJsonSerializer.Serialize(advisory); + var indented = CanonicalJsonSerializer.SerializeIndented(advisory); + results.Add((name, compact, indented)); + } + + return results; + } + finally + { + CultureInfo.CurrentCulture = originalCulture; + CultureInfo.CurrentUICulture = originalUiCulture; + } + } +} diff --git a/src/StellaOps.Feedser.Models.Tests/SeverityNormalizationTests.cs b/src/StellaOps.Feedser.Models.Tests/SeverityNormalizationTests.cs new file mode 100644 index 00000000..04bc0f91 --- /dev/null +++ b/src/StellaOps.Feedser.Models.Tests/SeverityNormalizationTests.cs @@ -0,0 +1,36 @@ +using StellaOps.Feedser.Models; + +namespace StellaOps.Feedser.Models.Tests; + +public sealed class SeverityNormalizationTests +{ + [Theory] + [InlineData("CRITICAL", "critical")] + [InlineData("Important", "high")] + [InlineData("moderate", "medium")] + [InlineData("Minor", "low")] + [InlineData("Info", "informational")] + [InlineData("negligible", "none")] + [InlineData("unknown", "unknown")] + [InlineData("Sev Critical", "critical")] + [InlineData("HIGH vendor", "high")] + [InlineData("Informative", "informational")] + [InlineData("Not Applicable", "none")] + [InlineData("Undetermined", "unknown")] + [InlineData("Priority 0", "critical")] + [InlineData("Priority-2", "medium")] + [InlineData("N/A", "none")] + [InlineData("custom-level", "custom-level")] + public void Normalize_ReturnsExpectedCanonicalValue(string input, string expected) + { + var normalized = SeverityNormalization.Normalize(input); + Assert.Equal(expected, normalized); + } + + [Fact] + public void Normalize_ReturnsNullWhenInputNullOrWhitespace() + { + Assert.Null(SeverityNormalization.Normalize(null)); + Assert.Null(SeverityNormalization.Normalize(" ")); + } +} diff --git a/src/StellaOps.Feedser.Models.Tests/StellaOps.Feedser.Models.Tests.csproj b/src/StellaOps.Feedser.Models.Tests/StellaOps.Feedser.Models.Tests.csproj new file mode 100644 index 00000000..8a7e97a1 --- /dev/null +++ b/src/StellaOps.Feedser.Models.Tests/StellaOps.Feedser.Models.Tests.csproj @@ -0,0 +1,15 @@ + + + net10.0 + enable + enable + + + + + + + 1 + + + diff --git a/src/StellaOps.Feedser.Models/AGENTS.md b/src/StellaOps.Feedser.Models/AGENTS.md new file mode 100644 index 00000000..481ffaeb --- /dev/null +++ b/src/StellaOps.Feedser.Models/AGENTS.md @@ -0,0 +1,30 @@ +# AGENTS +## Role +Canonical data model for normalized advisories and all downstream serialization. Source of truth for merge/export. +## Scope +- Canonical types: Advisory, AdvisoryReference, CvssMetric, AffectedPackage, AffectedVersionRange, AdvisoryProvenance. +- Invariants: stable ordering, culture-invariant serialization, UTC timestamps, deterministic equality semantics. +- Field semantics: preserve all aliases/references; ranges per ecosystem (NEVRA/EVR/SemVer); provenance on every mapped field. +- Backward/forward compatibility: additive evolution; versioned DTOs where needed; no breaking field renames. +- Detailed field coverage documented in `CANONICAL_RECORDS.md`; update alongside model changes. +## Participants +- Source connectors map external DTOs into these types. +- Merge engine composes/overrides AffectedPackage sets and consolidates references/aliases. +- Exporters serialize canonical documents deterministically. +## Interfaces & contracts +- Null-object statics: Advisory.Empty, AdvisoryReference.Empty, CvssMetric.Empty. +- AffectedPackage.Type describes semantics (e.g., rpm, deb, cpe, semver). Identifier is stable (e.g., NEVRA, PURL, CPE). +- Version ranges list is ordered by introduction then fix; provenance identifies source/kind/value/recordedAt. +- Alias schemes must include CVE, GHSA, OSV, JVN/JVNDB, BDU, VU(CERT/CC), MSRC, CISCO-SA, ORACLE-CPU, APSB/APA, APPLE-HT, CHROMIUM-POST, VMSA, RHSA, USN, DSA, SUSE-SU, ICSA, CWE, CPE, PURL. +## In/Out of scope +In: data shapes, invariants, helpers for canonical serialization and comparison. +Out: fetching/parsing external schemas, storage, HTTP. +## Observability & security expectations +- No secrets; purely in-memory types. +- Provide debug renders for test snapshots (canonical JSON). +- Emit model version identifiers in logs when canonical structures change; keep adapters for older readers until deprecated. +## Tests +- Author and review coverage in `../StellaOps.Feedser.Models.Tests`. +- Shared fixtures (e.g., `MongoIntegrationFixture`, `ConnectorTestHarness`) live in `../StellaOps.Feedser.Testing`. +- Keep fixtures deterministic; match new cases to real-world advisories or regression scenarios. + diff --git a/src/StellaOps.Feedser.Models/Advisory.cs b/src/StellaOps.Feedser.Models/Advisory.cs new file mode 100644 index 00000000..c02dabee --- /dev/null +++ b/src/StellaOps.Feedser.Models/Advisory.cs @@ -0,0 +1,189 @@ +using System.Collections.Immutable; +using System.Linq; +using System.Text.Json.Serialization; + +namespace StellaOps.Feedser.Models; + +/// +/// Canonical advisory document produced after merge. Collections are pre-sorted for deterministic serialization. +/// +public sealed record Advisory +{ + public static Advisory Empty { get; } = new( + advisoryKey: "unknown", + title: "", + summary: null, + language: null, + published: null, + modified: null, + severity: null, + exploitKnown: false, + aliases: Array.Empty(), + credits: Array.Empty(), + references: Array.Empty(), + affectedPackages: Array.Empty(), + cvssMetrics: Array.Empty(), + provenance: Array.Empty()); + + public Advisory( + string advisoryKey, + string title, + string? summary, + string? language, + DateTimeOffset? published, + DateTimeOffset? modified, + string? severity, + bool exploitKnown, + IEnumerable? aliases, + IEnumerable? references, + IEnumerable? affectedPackages, + IEnumerable? cvssMetrics, + IEnumerable? provenance) + : this( + advisoryKey, + title, + summary, + language, + published, + modified, + severity, + exploitKnown, + aliases, + Array.Empty(), + references, + affectedPackages, + cvssMetrics, + provenance) + { + } + + public Advisory( + string advisoryKey, + string title, + string? summary, + string? language, + DateTimeOffset? published, + DateTimeOffset? modified, + string? severity, + bool exploitKnown, + IEnumerable? aliases, + IEnumerable? credits, + IEnumerable? references, + IEnumerable? affectedPackages, + IEnumerable? cvssMetrics, + IEnumerable? provenance) + { + AdvisoryKey = Validation.EnsureNotNullOrWhiteSpace(advisoryKey, nameof(advisoryKey)); + Title = Validation.EnsureNotNullOrWhiteSpace(title, nameof(title)); + Summary = Validation.TrimToNull(summary); + Language = Validation.TrimToNull(language)?.ToLowerInvariant(); + Published = published?.ToUniversalTime(); + Modified = modified?.ToUniversalTime(); + Severity = SeverityNormalization.Normalize(severity); + ExploitKnown = exploitKnown; + + Aliases = (aliases ?? Array.Empty()) + .Select(static alias => Validation.TryNormalizeAlias(alias, out var normalized) ? normalized! : null) + .Where(static alias => alias is not null) + .Distinct(StringComparer.Ordinal) + .OrderBy(static alias => alias, StringComparer.Ordinal) + .Select(static alias => alias!) + .ToImmutableArray(); + + Credits = (credits ?? Array.Empty()) + .Where(static credit => credit is not null) + .OrderBy(static credit => credit.Role, StringComparer.Ordinal) + .ThenBy(static credit => credit.DisplayName, StringComparer.Ordinal) + .ToImmutableArray(); + + References = (references ?? Array.Empty()) + .Where(static reference => reference is not null) + .OrderBy(static reference => reference.Url, StringComparer.Ordinal) + .ThenBy(static reference => reference.Kind, StringComparer.Ordinal) + .ThenBy(static reference => reference.SourceTag, StringComparer.Ordinal) + .ThenBy(static reference => reference.Provenance.RecordedAt) + .ToImmutableArray(); + + AffectedPackages = (affectedPackages ?? Array.Empty()) + .Where(static package => package is not null) + .OrderBy(static package => package.Type, StringComparer.Ordinal) + .ThenBy(static package => package.Identifier, StringComparer.Ordinal) + .ThenBy(static package => package.Platform, StringComparer.Ordinal) + .ToImmutableArray(); + + CvssMetrics = (cvssMetrics ?? Array.Empty()) + .Where(static metric => metric is not null) + .OrderBy(static metric => metric.Version, StringComparer.Ordinal) + .ThenBy(static metric => metric.Vector, StringComparer.Ordinal) + .ToImmutableArray(); + + Provenance = (provenance ?? Array.Empty()) + .Where(static p => p is not null) + .OrderBy(static p => p.Source, StringComparer.Ordinal) + .ThenBy(static p => p.Kind, StringComparer.Ordinal) + .ThenBy(static p => p.RecordedAt) + .ToImmutableArray(); + } + + [JsonConstructor] + public Advisory( + string advisoryKey, + string title, + string? summary, + string? language, + DateTimeOffset? published, + DateTimeOffset? modified, + string? severity, + bool exploitKnown, + ImmutableArray aliases, + ImmutableArray credits, + ImmutableArray references, + ImmutableArray affectedPackages, + ImmutableArray cvssMetrics, + ImmutableArray provenance) + : this( + advisoryKey, + title, + summary, + language, + published, + modified, + severity, + exploitKnown, + aliases.IsDefault ? null : aliases.AsEnumerable(), + credits.IsDefault ? null : credits.AsEnumerable(), + references.IsDefault ? null : references.AsEnumerable(), + affectedPackages.IsDefault ? null : affectedPackages.AsEnumerable(), + cvssMetrics.IsDefault ? null : cvssMetrics.AsEnumerable(), + provenance.IsDefault ? null : provenance.AsEnumerable()) + { + } + + public string AdvisoryKey { get; } + + public string Title { get; } + + public string? Summary { get; } + + public string? Language { get; } + + public DateTimeOffset? Published { get; } + + public DateTimeOffset? Modified { get; } + + public string? Severity { get; } + + public bool ExploitKnown { get; } + + public ImmutableArray Aliases { get; } + + public ImmutableArray Credits { get; } + + public ImmutableArray References { get; } + + public ImmutableArray AffectedPackages { get; } + + public ImmutableArray CvssMetrics { get; } + + public ImmutableArray Provenance { get; } +} diff --git a/src/StellaOps.Feedser.Models/AdvisoryCredit.cs b/src/StellaOps.Feedser.Models/AdvisoryCredit.cs new file mode 100644 index 00000000..e4252410 --- /dev/null +++ b/src/StellaOps.Feedser.Models/AdvisoryCredit.cs @@ -0,0 +1,101 @@ +using System.Collections.Generic; +using System.Collections.Immutable; +using System.Linq; +using System.Text; +using System.Text.Json.Serialization; + +namespace StellaOps.Feedser.Models; + +/// +/// Canonical acknowledgement/credit metadata associated with an advisory. +/// +public sealed record AdvisoryCredit +{ + public static AdvisoryCredit Empty { get; } = new("unknown", role: null, contacts: Array.Empty(), AdvisoryProvenance.Empty); + + [JsonConstructor] + public AdvisoryCredit(string displayName, string? role, ImmutableArray contacts, AdvisoryProvenance provenance) + : this(displayName, role, contacts.IsDefault ? null : contacts.AsEnumerable(), provenance) + { + } + + public AdvisoryCredit(string displayName, string? role, IEnumerable? contacts, AdvisoryProvenance provenance) + { + DisplayName = Validation.EnsureNotNullOrWhiteSpace(displayName, nameof(displayName)); + Role = NormalizeRole(role); + Contacts = NormalizeContacts(contacts); + Provenance = provenance ?? AdvisoryProvenance.Empty; + } + + public string DisplayName { get; } + + public string? Role { get; } + + public ImmutableArray Contacts { get; } + + public AdvisoryProvenance Provenance { get; } + + private static string? NormalizeRole(string? role) + { + if (string.IsNullOrWhiteSpace(role)) + { + return null; + } + + var span = role.AsSpan(); + var buffer = new StringBuilder(span.Length); + + foreach (var ch in span) + { + if (char.IsLetterOrDigit(ch)) + { + buffer.Append(char.ToLowerInvariant(ch)); + continue; + } + + if (ch is '-' or '_' or ' ') + { + if (buffer.Length > 0 && buffer[^1] != '_') + { + buffer.Append('_'); + } + + continue; + } + } + + while (buffer.Length > 0 && buffer[^1] == '_') + { + buffer.Length--; + } + + return buffer.Length == 0 ? null : buffer.ToString(); + } + + private static ImmutableArray NormalizeContacts(IEnumerable? contacts) + { + if (contacts is null) + { + return ImmutableArray.Empty; + } + + var set = new SortedSet(StringComparer.Ordinal); + foreach (var contact in contacts) + { + if (string.IsNullOrWhiteSpace(contact)) + { + continue; + } + + var trimmed = contact.Trim(); + if (trimmed.Length == 0) + { + continue; + } + + set.Add(trimmed); + } + + return set.Count == 0 ? ImmutableArray.Empty : set.ToImmutableArray(); + } +} diff --git a/src/StellaOps.Feedser.Models/AdvisoryProvenance.cs b/src/StellaOps.Feedser.Models/AdvisoryProvenance.cs new file mode 100644 index 00000000..d70d4dce --- /dev/null +++ b/src/StellaOps.Feedser.Models/AdvisoryProvenance.cs @@ -0,0 +1,70 @@ +using System.Collections.Immutable; +using System.Linq; +using System.Text.Json.Serialization; + +namespace StellaOps.Feedser.Models; + +/// +/// Describes the origin of a canonical field and how/when it was captured. +/// +public sealed record AdvisoryProvenance +{ + public static AdvisoryProvenance Empty { get; } = new("unknown", "unspecified", string.Empty, DateTimeOffset.UnixEpoch); + + [JsonConstructor] + public AdvisoryProvenance( + string source, + string kind, + string value, + string? decisionReason, + DateTimeOffset recordedAt, + ImmutableArray fieldMask) + : this(source, kind, value, recordedAt, fieldMask.IsDefault ? null : fieldMask.AsEnumerable(), decisionReason) + { + } + + public AdvisoryProvenance( + string source, + string kind, + string value, + DateTimeOffset recordedAt, + IEnumerable? fieldMask = null, + string? decisionReason = null) + { + Source = Validation.EnsureNotNullOrWhiteSpace(source, nameof(source)); + Kind = Validation.EnsureNotNullOrWhiteSpace(kind, nameof(kind)); + Value = Validation.TrimToNull(value); + DecisionReason = Validation.TrimToNull(decisionReason); + RecordedAt = recordedAt.ToUniversalTime(); + FieldMask = NormalizeFieldMask(fieldMask); + } + + public string Source { get; } + + public string Kind { get; } + + public string? Value { get; } + + public string? DecisionReason { get; } + + public DateTimeOffset RecordedAt { get; } + + public ImmutableArray FieldMask { get; } + + private static ImmutableArray NormalizeFieldMask(IEnumerable? fieldMask) + { + if (fieldMask is null) + { + return ImmutableArray.Empty; + } + + var buffer = fieldMask + .Where(static value => !string.IsNullOrWhiteSpace(value)) + .Select(static value => value.Trim().ToLowerInvariant()) + .Distinct(StringComparer.Ordinal) + .OrderBy(static value => value, StringComparer.Ordinal) + .ToImmutableArray(); + + return buffer.IsDefault ? ImmutableArray.Empty : buffer; + } +} diff --git a/src/StellaOps.Feedser.Models/AdvisoryReference.cs b/src/StellaOps.Feedser.Models/AdvisoryReference.cs new file mode 100644 index 00000000..e173745c --- /dev/null +++ b/src/StellaOps.Feedser.Models/AdvisoryReference.cs @@ -0,0 +1,36 @@ +using System.Text.Json.Serialization; + +namespace StellaOps.Feedser.Models; + +/// +/// Canonical external reference associated with an advisory. +/// +public sealed record AdvisoryReference +{ + public static AdvisoryReference Empty { get; } = new("https://invalid.local/", kind: null, sourceTag: null, summary: null, provenance: AdvisoryProvenance.Empty); + + [JsonConstructor] + public AdvisoryReference(string url, string? kind, string? sourceTag, string? summary, AdvisoryProvenance provenance) + { + if (!Validation.LooksLikeHttpUrl(url)) + { + throw new ArgumentException("Reference URL must be an absolute http(s) URI.", nameof(url)); + } + + Url = url; + Kind = Validation.TrimToNull(kind); + SourceTag = Validation.TrimToNull(sourceTag); + Summary = Validation.TrimToNull(summary); + Provenance = provenance ?? AdvisoryProvenance.Empty; + } + + public string Url { get; } + + public string? Kind { get; } + + public string? SourceTag { get; } + + public string? Summary { get; } + + public AdvisoryProvenance Provenance { get; } +} diff --git a/src/StellaOps.Feedser.Models/AffectedPackage.cs b/src/StellaOps.Feedser.Models/AffectedPackage.cs new file mode 100644 index 00000000..a305ae02 --- /dev/null +++ b/src/StellaOps.Feedser.Models/AffectedPackage.cs @@ -0,0 +1,97 @@ +using System; +using System.Collections.Generic; +using System.Collections.Immutable; +using System.Linq; +using System.Text.Json.Serialization; + +namespace StellaOps.Feedser.Models; + +/// +/// Canonical affected package descriptor with deterministic ordering of ranges and provenance. +/// +public sealed record AffectedPackage +{ + public static AffectedPackage Empty { get; } = new( + AffectedPackageTypes.SemVer, + identifier: "unknown", + platform: null, + versionRanges: Array.Empty(), + statuses: Array.Empty(), + provenance: Array.Empty(), + normalizedVersions: Array.Empty()); + + [JsonConstructor] + public AffectedPackage( + string type, + string identifier, + string? platform = null, + IEnumerable? versionRanges = null, + IEnumerable? statuses = null, + IEnumerable? provenance = null, + IEnumerable? normalizedVersions = null) + { + Type = Validation.EnsureNotNullOrWhiteSpace(type, nameof(type)).ToLowerInvariant(); + Identifier = Validation.EnsureNotNullOrWhiteSpace(identifier, nameof(identifier)); + Platform = Validation.TrimToNull(platform); + + VersionRanges = (versionRanges ?? Array.Empty()) + .Distinct(AffectedVersionRangeEqualityComparer.Instance) + .OrderBy(static range => range, AffectedVersionRangeComparer.Instance) + .ToImmutableArray(); + + Statuses = (statuses ?? Array.Empty()) + .Where(static status => status is not null) + .Distinct(AffectedPackageStatusEqualityComparer.Instance) + .OrderBy(static status => status.Status, StringComparer.Ordinal) + .ThenBy(static status => status.Provenance.Source, StringComparer.Ordinal) + .ThenBy(static status => status.Provenance.Kind, StringComparer.Ordinal) + .ThenBy(static status => status.Provenance.RecordedAt) + .ToImmutableArray(); + + NormalizedVersions = (normalizedVersions ?? Array.Empty()) + .Where(static rule => rule is not null) + .Distinct(NormalizedVersionRuleEqualityComparer.Instance) + .OrderBy(static rule => rule, NormalizedVersionRuleComparer.Instance) + .ToImmutableArray(); + + Provenance = (provenance ?? Array.Empty()) + .Where(static p => p is not null) + .OrderBy(static p => p.Source, StringComparer.Ordinal) + .ThenBy(static p => p.Kind, StringComparer.Ordinal) + .ThenBy(static p => p.RecordedAt) + .ToImmutableArray(); + } + + /// + /// Semantic type of the coordinates (rpm, deb, cpe, semver, vendor, ics-vendor). + /// + public string Type { get; } + + /// + /// Canonical identifier for the package (NEVRA, PackageURL, CPE string, vendor slug, etc.). + /// + public string Identifier { get; } + + public string? Platform { get; } + + public ImmutableArray VersionRanges { get; } + + public ImmutableArray Statuses { get; } + + public ImmutableArray NormalizedVersions { get; } + + public ImmutableArray Provenance { get; } +} + +/// +/// Known values for . +/// +public static class AffectedPackageTypes +{ + public const string Rpm = "rpm"; + public const string Deb = "deb"; + public const string Cpe = "cpe"; + public const string SemVer = "semver"; + public const string Vendor = "vendor"; + public const string IcsVendor = "ics-vendor"; +} diff --git a/src/StellaOps.Feedser.Models/AffectedPackageStatus.cs b/src/StellaOps.Feedser.Models/AffectedPackageStatus.cs new file mode 100644 index 00000000..5ca773b7 --- /dev/null +++ b/src/StellaOps.Feedser.Models/AffectedPackageStatus.cs @@ -0,0 +1,46 @@ +using System; +using System.Collections.Generic; +using System.Text.Json.Serialization; + +namespace StellaOps.Feedser.Models; + +/// +/// Represents a vendor-supplied status tag for an affected package when a concrete version range is unavailable or supplementary. +/// +public sealed record AffectedPackageStatus +{ + [JsonConstructor] + public AffectedPackageStatus(string status, AdvisoryProvenance provenance) + { + Status = AffectedPackageStatusCatalog.Normalize(status); + Provenance = provenance ?? AdvisoryProvenance.Empty; + } + + public string Status { get; } + + public AdvisoryProvenance Provenance { get; } +} + +public sealed class AffectedPackageStatusEqualityComparer : IEqualityComparer +{ + public static AffectedPackageStatusEqualityComparer Instance { get; } = new(); + + public bool Equals(AffectedPackageStatus? x, AffectedPackageStatus? y) + { + if (ReferenceEquals(x, y)) + { + return true; + } + + if (x is null || y is null) + { + return false; + } + + return string.Equals(x.Status, y.Status, StringComparison.Ordinal) + && EqualityComparer.Default.Equals(x.Provenance, y.Provenance); + } + + public int GetHashCode(AffectedPackageStatus obj) + => HashCode.Combine(obj.Status, obj.Provenance); +} diff --git a/src/StellaOps.Feedser.Models/AffectedPackageStatusCatalog.cs b/src/StellaOps.Feedser.Models/AffectedPackageStatusCatalog.cs new file mode 100644 index 00000000..18ad4ad8 --- /dev/null +++ b/src/StellaOps.Feedser.Models/AffectedPackageStatusCatalog.cs @@ -0,0 +1,156 @@ +using System; +using System.Collections.Generic; +using System.Diagnostics.CodeAnalysis; + +namespace StellaOps.Feedser.Models; + +/// +/// Central registry of allowed affected-package status labels to keep connectors consistent. +/// +public static class AffectedPackageStatusCatalog +{ + public const string KnownAffected = "known_affected"; + public const string KnownNotAffected = "known_not_affected"; + public const string UnderInvestigation = "under_investigation"; + public const string Fixed = "fixed"; + public const string FirstFixed = "first_fixed"; + public const string Mitigated = "mitigated"; + public const string NotApplicable = "not_applicable"; + public const string Affected = "affected"; + public const string NotAffected = "not_affected"; + public const string Pending = "pending"; + public const string Unknown = "unknown"; + + private static readonly string[] CanonicalStatuses = + { + KnownAffected, + KnownNotAffected, + UnderInvestigation, + Fixed, + FirstFixed, + Mitigated, + NotApplicable, + Affected, + NotAffected, + Pending, + Unknown, + }; + + private static readonly IReadOnlyList AllowedStatuses = Array.AsReadOnly(CanonicalStatuses); + + private static readonly IReadOnlyDictionary StatusMap = BuildStatusMap(); + + public static IReadOnlyList Allowed => AllowedStatuses; + + public static string Normalize(string status) + { + if (!TryNormalize(status, out var normalized)) + { + throw new ArgumentOutOfRangeException(nameof(status), status, "Status is not part of the allowed affected-package status glossary."); + } + + return normalized; + } + + public static bool TryNormalize(string? status, [NotNullWhen(true)] out string? normalized) + { + normalized = null; + + if (string.IsNullOrWhiteSpace(status)) + { + return false; + } + + var token = Sanitize(status); + if (token.Length == 0) + { + return false; + } + + if (!StatusMap.TryGetValue(token, out normalized)) + { + return false; + } + + return true; + } + + public static bool IsAllowed(string? status) + => TryNormalize(status, out _); + + private static IReadOnlyDictionary BuildStatusMap() + { + var map = new Dictionary(StringComparer.Ordinal); + foreach (var status in CanonicalStatuses) + { + map[Sanitize(status)] = status; + } + + Add(map, "known not vulnerable", KnownNotAffected); + Add(map, "known unaffected", KnownNotAffected); + Add(map, "known not impacted", KnownNotAffected); + Add(map, "vulnerable", Affected); + Add(map, "impacted", Affected); + Add(map, "impacting", Affected); + Add(map, "not vulnerable", NotAffected); + Add(map, "unaffected", NotAffected); + Add(map, "not impacted", NotAffected); + Add(map, "no impact", NotAffected); + Add(map, "impact free", NotAffected); + Add(map, "investigating", UnderInvestigation); + Add(map, "analysis in progress", UnderInvestigation); + Add(map, "analysis pending", UnderInvestigation); + Add(map, "patch available", Fixed); + Add(map, "fix available", Fixed); + Add(map, "patched", Fixed); + Add(map, "resolved", Fixed); + Add(map, "remediated", Fixed); + Add(map, "workaround available", Mitigated); + Add(map, "mitigation available", Mitigated); + Add(map, "mitigation provided", Mitigated); + Add(map, "not applicable", NotApplicable); + Add(map, "n/a", NotApplicable); + Add(map, "na", NotApplicable); + Add(map, "does not apply", NotApplicable); + Add(map, "out of scope", NotApplicable); + Add(map, "pending fix", Pending); + Add(map, "awaiting fix", Pending); + Add(map, "awaiting patch", Pending); + Add(map, "scheduled", Pending); + Add(map, "planned", Pending); + Add(map, "tbd", Unknown); + Add(map, "to be determined", Unknown); + Add(map, "undetermined", Unknown); + Add(map, "not yet known", Unknown); + + return map; + } + + private static void Add(IDictionary map, string alias, string canonical) + { + var key = Sanitize(alias); + if (key.Length == 0) + { + return; + } + + map[key] = canonical; + } + + private static string Sanitize(string value) + { + var span = value.AsSpan(); + var buffer = new char[span.Length]; + var index = 0; + + foreach (var ch in span) + { + if (char.IsLetterOrDigit(ch)) + { + buffer[index++] = char.ToLowerInvariant(ch); + } + } + + return index == 0 ? string.Empty : new string(buffer, 0, index); + } +} diff --git a/src/StellaOps.Feedser.Models/AffectedVersionRange.cs b/src/StellaOps.Feedser.Models/AffectedVersionRange.cs new file mode 100644 index 00000000..a71e1224 --- /dev/null +++ b/src/StellaOps.Feedser.Models/AffectedVersionRange.cs @@ -0,0 +1,149 @@ +using System.Text.Json.Serialization; + +namespace StellaOps.Feedser.Models; + +/// +/// Describes a contiguous range of versions impacted by an advisory. +/// +public sealed record AffectedVersionRange +{ + [JsonConstructor] + public AffectedVersionRange( + string rangeKind, + string? introducedVersion, + string? fixedVersion, + string? lastAffectedVersion, + string? rangeExpression, + AdvisoryProvenance provenance, + RangePrimitives? primitives = null) + { + RangeKind = Validation.EnsureNotNullOrWhiteSpace(rangeKind, nameof(rangeKind)).ToLowerInvariant(); + IntroducedVersion = Validation.TrimToNull(introducedVersion); + FixedVersion = Validation.TrimToNull(fixedVersion); + LastAffectedVersion = Validation.TrimToNull(lastAffectedVersion); + RangeExpression = Validation.TrimToNull(rangeExpression); + Provenance = provenance ?? AdvisoryProvenance.Empty; + Primitives = primitives; + } + + /// + /// Semantic kind of the range (e.g., semver, nevra, evr). + /// + public string RangeKind { get; } + + /// + /// Inclusive version where impact begins. + /// + public string? IntroducedVersion { get; } + + /// + /// Exclusive version where impact ends due to a fix. + /// + public string? FixedVersion { get; } + + /// + /// Inclusive upper bound where the vendor reports exposure (when no fix available). + /// + public string? LastAffectedVersion { get; } + + /// + /// Normalized textual representation of the range (fallback). + /// + public string? RangeExpression { get; } + + public AdvisoryProvenance Provenance { get; } + + public RangePrimitives? Primitives { get; } + + public string CreateDeterministicKey() + => string.Join('|', RangeKind, IntroducedVersion ?? string.Empty, FixedVersion ?? string.Empty, LastAffectedVersion ?? string.Empty, RangeExpression ?? string.Empty); +} + +/// +/// Deterministic comparer for version ranges. Orders by introduced, fixed, last affected, expression, kind. +/// +public sealed class AffectedVersionRangeComparer : IComparer +{ + public static AffectedVersionRangeComparer Instance { get; } = new(); + + private static readonly StringComparer Comparer = StringComparer.Ordinal; + + public int Compare(AffectedVersionRange? x, AffectedVersionRange? y) + { + if (ReferenceEquals(x, y)) + { + return 0; + } + + if (x is null) + { + return -1; + } + + if (y is null) + { + return 1; + } + + var compare = Comparer.Compare(x.IntroducedVersion, y.IntroducedVersion); + if (compare != 0) + { + return compare; + } + + compare = Comparer.Compare(x.FixedVersion, y.FixedVersion); + if (compare != 0) + { + return compare; + } + + compare = Comparer.Compare(x.LastAffectedVersion, y.LastAffectedVersion); + if (compare != 0) + { + return compare; + } + + compare = Comparer.Compare(x.RangeExpression, y.RangeExpression); + if (compare != 0) + { + return compare; + } + + return Comparer.Compare(x.RangeKind, y.RangeKind); + } +} + +/// +/// Equality comparer that ignores provenance differences. +/// +public sealed class AffectedVersionRangeEqualityComparer : IEqualityComparer +{ + public static AffectedVersionRangeEqualityComparer Instance { get; } = new(); + + public bool Equals(AffectedVersionRange? x, AffectedVersionRange? y) + { + if (ReferenceEquals(x, y)) + { + return true; + } + + if (x is null || y is null) + { + return false; + } + + return string.Equals(x.RangeKind, y.RangeKind, StringComparison.Ordinal) + && string.Equals(x.IntroducedVersion, y.IntroducedVersion, StringComparison.Ordinal) + && string.Equals(x.FixedVersion, y.FixedVersion, StringComparison.Ordinal) + && string.Equals(x.LastAffectedVersion, y.LastAffectedVersion, StringComparison.Ordinal) + && string.Equals(x.RangeExpression, y.RangeExpression, StringComparison.Ordinal); + } + + public int GetHashCode(AffectedVersionRange obj) + => HashCode.Combine( + obj.RangeKind, + obj.IntroducedVersion, + obj.FixedVersion, + obj.LastAffectedVersion, + obj.RangeExpression); +} diff --git a/src/StellaOps.Feedser.Models/AffectedVersionRangeExtensions.cs b/src/StellaOps.Feedser.Models/AffectedVersionRangeExtensions.cs new file mode 100644 index 00000000..045df114 --- /dev/null +++ b/src/StellaOps.Feedser.Models/AffectedVersionRangeExtensions.cs @@ -0,0 +1,75 @@ +using System; + +namespace StellaOps.Feedser.Models; + +/// +/// Helpers for deriving normalized version rules from affected version ranges. +/// +public static class AffectedVersionRangeExtensions +{ + public static NormalizedVersionRule? ToNormalizedVersionRule(this AffectedVersionRange? range, string? notes = null) + { + if (range is null) + { + return null; + } + + var semVer = range.Primitives?.SemVer; + if (semVer is not null) + { + var rule = semVer.ToNormalizedVersionRule(notes); + if (rule is not null) + { + return rule; + } + } + + if (!string.Equals(range.RangeKind, NormalizedVersionSchemes.SemVer, StringComparison.OrdinalIgnoreCase)) + { + return null; + } + + var min = Validation.TrimToNull(range.IntroducedVersion); + var max = Validation.TrimToNull(range.FixedVersion); + var last = Validation.TrimToNull(range.LastAffectedVersion); + + if (string.IsNullOrEmpty(min) && string.IsNullOrEmpty(max) && string.IsNullOrEmpty(last)) + { + return null; + } + + if (!string.IsNullOrEmpty(max)) + { + return new NormalizedVersionRule( + NormalizedVersionSchemes.SemVer, + NormalizedVersionRuleTypes.Range, + min: min, + minInclusive: min is null ? null : true, + max: max, + maxInclusive: false, + notes: notes); + } + + if (!string.IsNullOrEmpty(last)) + { + return new NormalizedVersionRule( + NormalizedVersionSchemes.SemVer, + NormalizedVersionRuleTypes.LessThanOrEqual, + max: last, + maxInclusive: true, + notes: notes); + } + + if (!string.IsNullOrEmpty(min)) + { + return new NormalizedVersionRule( + NormalizedVersionSchemes.SemVer, + NormalizedVersionRuleTypes.GreaterThanOrEqual, + min: min, + minInclusive: true, + notes: notes); + } + + return null; + } +} diff --git a/src/StellaOps.Feedser.Models/AliasSchemeRegistry.cs b/src/StellaOps.Feedser.Models/AliasSchemeRegistry.cs new file mode 100644 index 00000000..5e5c01e5 --- /dev/null +++ b/src/StellaOps.Feedser.Models/AliasSchemeRegistry.cs @@ -0,0 +1,166 @@ +using System; +using System.Collections.Generic; +using System.Collections.Immutable; +using System.Linq; +using System.Text.RegularExpressions; + +namespace StellaOps.Feedser.Models; + +public static class AliasSchemeRegistry +{ + private sealed record AliasScheme( + string Name, + Func Predicate, + Func Normalizer); + +private static readonly AliasScheme[] SchemeDefinitions = + { + BuildScheme(AliasSchemes.Cve, alias => alias is not null && Matches(CvERegex, alias), alias => alias is null ? string.Empty : NormalizePrefix(alias, "CVE")), + BuildScheme(AliasSchemes.Ghsa, alias => alias is not null && Matches(GhsaRegex, alias), alias => alias is null ? string.Empty : NormalizePrefix(alias, "GHSA")), + BuildScheme(AliasSchemes.OsV, alias => alias is not null && Matches(OsVRegex, alias), alias => alias is null ? string.Empty : NormalizePrefix(alias, "OSV")), + BuildScheme(AliasSchemes.Jvn, alias => alias is not null && Matches(JvnRegex, alias), alias => alias is null ? string.Empty : NormalizePrefix(alias, "JVN")), + BuildScheme(AliasSchemes.Jvndb, alias => alias is not null && Matches(JvndbRegex, alias), alias => alias is null ? string.Empty : NormalizePrefix(alias, "JVNDB")), + BuildScheme(AliasSchemes.Bdu, alias => alias is not null && Matches(BduRegex, alias), alias => alias is null ? string.Empty : NormalizePrefix(alias, "BDU")), + BuildScheme(AliasSchemes.Vu, alias => alias is not null && alias.StartsWith("VU#", StringComparison.OrdinalIgnoreCase), alias => NormalizePrefix(alias, "VU", preserveSeparator: '#')), + BuildScheme(AliasSchemes.Msrc, alias => alias is not null && alias.StartsWith("MSRC-", StringComparison.OrdinalIgnoreCase), alias => NormalizePrefix(alias, "MSRC")), + BuildScheme(AliasSchemes.CiscoSa, alias => alias is not null && alias.StartsWith("CISCO-SA-", StringComparison.OrdinalIgnoreCase), alias => NormalizePrefix(alias, "CISCO-SA")), + BuildScheme(AliasSchemes.OracleCpu, alias => alias is not null && alias.StartsWith("ORACLE-CPU", StringComparison.OrdinalIgnoreCase), alias => NormalizePrefix(alias, "ORACLE-CPU")), + BuildScheme(AliasSchemes.Apsb, alias => alias is not null && alias.StartsWith("APSB-", StringComparison.OrdinalIgnoreCase), alias => NormalizePrefix(alias, "APSB")), + BuildScheme(AliasSchemes.Apa, alias => alias is not null && alias.StartsWith("APA-", StringComparison.OrdinalIgnoreCase), alias => NormalizePrefix(alias, "APA")), + BuildScheme(AliasSchemes.AppleHt, alias => alias is not null && alias.StartsWith("APPLE-HT", StringComparison.OrdinalIgnoreCase), alias => NormalizePrefix(alias, "APPLE-HT")), + BuildScheme(AliasSchemes.ChromiumPost, alias => alias is not null && (alias.StartsWith("CHROMIUM-POST", StringComparison.OrdinalIgnoreCase) || alias.StartsWith("CHROMIUM:", StringComparison.OrdinalIgnoreCase)), NormalizeChromium), + BuildScheme(AliasSchemes.Vmsa, alias => alias is not null && alias.StartsWith("VMSA-", StringComparison.OrdinalIgnoreCase), alias => NormalizePrefix(alias, "VMSA")), + BuildScheme(AliasSchemes.Rhsa, alias => alias is not null && alias.StartsWith("RHSA-", StringComparison.OrdinalIgnoreCase), alias => NormalizePrefix(alias, "RHSA")), + BuildScheme(AliasSchemes.Usn, alias => alias is not null && alias.StartsWith("USN-", StringComparison.OrdinalIgnoreCase), alias => NormalizePrefix(alias, "USN")), + BuildScheme(AliasSchemes.Dsa, alias => alias is not null && alias.StartsWith("DSA-", StringComparison.OrdinalIgnoreCase), alias => NormalizePrefix(alias, "DSA")), + BuildScheme(AliasSchemes.SuseSu, alias => alias is not null && alias.StartsWith("SUSE-SU-", StringComparison.OrdinalIgnoreCase), alias => NormalizePrefix(alias, "SUSE-SU")), + BuildScheme(AliasSchemes.Icsa, alias => alias is not null && alias.StartsWith("ICSA-", StringComparison.OrdinalIgnoreCase), alias => NormalizePrefix(alias, "ICSA")), + BuildScheme(AliasSchemes.Cwe, alias => alias is not null && Matches(CweRegex, alias), alias => alias is null ? string.Empty : NormalizePrefix(alias, "CWE")), + BuildScheme(AliasSchemes.Cpe, alias => alias is not null && alias.StartsWith("cpe:", StringComparison.OrdinalIgnoreCase), alias => NormalizePrefix(alias, "cpe", uppercase:false)), + BuildScheme(AliasSchemes.Purl, alias => alias is not null && alias.StartsWith("pkg:", StringComparison.OrdinalIgnoreCase), alias => NormalizePrefix(alias, "pkg", uppercase:false)), + }; + + private static AliasScheme BuildScheme(string name, Func predicate, Func normalizer) + => new( + name, + predicate, + alias => normalizer(alias)); + + private static readonly ImmutableHashSet SchemeNames = SchemeDefinitions + .Select(static scheme => scheme.Name) + .ToImmutableHashSet(StringComparer.OrdinalIgnoreCase); + + private static readonly Regex CvERegex = new("^CVE-\\d{4}-\\d{4,}$", RegexOptions.Compiled | RegexOptions.CultureInvariant | RegexOptions.IgnoreCase); + private static readonly Regex GhsaRegex = new("^GHSA-[0-9a-z]{4}-[0-9a-z]{4}-[0-9a-z]{4}$", RegexOptions.Compiled | RegexOptions.CultureInvariant | RegexOptions.IgnoreCase); + private static readonly Regex OsVRegex = new("^OSV-\\d{4}-\\d+$", RegexOptions.Compiled | RegexOptions.CultureInvariant | RegexOptions.IgnoreCase); + private static readonly Regex JvnRegex = new("^JVN-\\d{4}-\\d{6}$", RegexOptions.Compiled | RegexOptions.CultureInvariant | RegexOptions.IgnoreCase); + private static readonly Regex JvndbRegex = new("^JVNDB-\\d{4}-\\d{6}$", RegexOptions.Compiled | RegexOptions.CultureInvariant | RegexOptions.IgnoreCase); + private static readonly Regex BduRegex = new("^BDU-\\d{4}-\\d+$", RegexOptions.Compiled | RegexOptions.CultureInvariant | RegexOptions.IgnoreCase); + private static readonly Regex CweRegex = new("^CWE-\\d+$", RegexOptions.Compiled | RegexOptions.CultureInvariant | RegexOptions.IgnoreCase); + + public static IReadOnlyCollection KnownSchemes => SchemeNames; + + public static bool IsKnownScheme(string? scheme) + => !string.IsNullOrWhiteSpace(scheme) && SchemeNames.Contains(scheme); + + public static bool TryGetScheme(string? alias, out string scheme) + { + if (string.IsNullOrWhiteSpace(alias)) + { + scheme = string.Empty; + return false; + } + + var candidate = alias.Trim(); + foreach (var entry in SchemeDefinitions) + { + if (entry.Predicate(candidate)) + { + scheme = entry.Name; + return true; + } + } + + scheme = string.Empty; + return false; + } + + public static bool TryNormalize(string? alias, out string normalized, out string scheme) + { + normalized = string.Empty; + scheme = string.Empty; + + if (string.IsNullOrWhiteSpace(alias)) + { + return false; + } + + var candidate = alias.Trim(); + foreach (var entry in SchemeDefinitions) + { + if (entry.Predicate(candidate)) + { + scheme = entry.Name; + normalized = entry.Normalizer(candidate); + return true; + } + } + + normalized = candidate; + return false; + } + + private static string NormalizePrefix(string? alias, string prefix, bool uppercase = true, char? preserveSeparator = null) + { + if (string.IsNullOrWhiteSpace(alias)) + { + return string.Empty; + } + + var comparison = StringComparison.OrdinalIgnoreCase; + if (!alias.StartsWith(prefix, comparison)) + { + return uppercase ? alias : alias.ToLowerInvariant(); + } + + var remainder = alias[prefix.Length..]; + if (preserveSeparator is { } separator && remainder.Length > 0 && remainder[0] != separator) + { + // Edge case: alias is expected to use a specific separator but does not – return unchanged. + return uppercase ? prefix.ToUpperInvariant() + remainder : prefix + remainder; + } + + var normalizedPrefix = uppercase ? prefix.ToUpperInvariant() : prefix.ToLowerInvariant(); + return normalizedPrefix + remainder; + } + + private static string NormalizeChromium(string? alias) + { + if (string.IsNullOrWhiteSpace(alias)) + { + return string.Empty; + } + + if (alias.StartsWith("CHROMIUM-POST", StringComparison.OrdinalIgnoreCase)) + { + return NormalizePrefix(alias, "CHROMIUM-POST"); + } + + if (alias.StartsWith("CHROMIUM:", StringComparison.OrdinalIgnoreCase)) + { + var remainder = alias["CHROMIUM".Length..]; + return "CHROMIUM" + remainder; + } + + return alias; + } + private static bool Matches(Regex? regex, string? candidate) + { + if (regex is null || string.IsNullOrWhiteSpace(candidate)) + { + return false; + } + + return regex.IsMatch(candidate); + } +} diff --git a/src/StellaOps.Feedser.Models/AliasSchemes.cs b/src/StellaOps.Feedser.Models/AliasSchemes.cs new file mode 100644 index 00000000..8227212a --- /dev/null +++ b/src/StellaOps.Feedser.Models/AliasSchemes.cs @@ -0,0 +1,31 @@ +namespace StellaOps.Feedser.Models; + +/// +/// Well-known alias scheme identifiers referenced throughout the pipeline. +/// +public static class AliasSchemes +{ + public const string Cve = "CVE"; + public const string Ghsa = "GHSA"; + public const string OsV = "OSV"; + public const string Jvn = "JVN"; + public const string Jvndb = "JVNDB"; + public const string Bdu = "BDU"; + public const string Vu = "VU"; + public const string Msrc = "MSRC"; + public const string CiscoSa = "CISCO-SA"; + public const string OracleCpu = "ORACLE-CPU"; + public const string Apsb = "APSB"; + public const string Apa = "APA"; + public const string AppleHt = "APPLE-HT"; + public const string ChromiumPost = "CHROMIUM-POST"; + public const string Vmsa = "VMSA"; + public const string Rhsa = "RHSA"; + public const string Usn = "USN"; + public const string Dsa = "DSA"; + public const string SuseSu = "SUSE-SU"; + public const string Icsa = "ICSA"; + public const string Cwe = "CWE"; + public const string Cpe = "CPE"; + public const string Purl = "PURL"; +} diff --git a/src/StellaOps.Feedser.Models/BACKWARD_COMPATIBILITY.md b/src/StellaOps.Feedser.Models/BACKWARD_COMPATIBILITY.md new file mode 100644 index 00000000..5cee4f67 --- /dev/null +++ b/src/StellaOps.Feedser.Models/BACKWARD_COMPATIBILITY.md @@ -0,0 +1,41 @@ +# Canonical Model Backward-Compatibility Playbook + +This playbook captures the policies and workflow required when evolving the canonical +`StellaOps.Feedser.Models` surface. + +## Principles + +- **Additive by default** – breaking field removals/renames are not allowed without a staged + migration plan. +- **Version-the-writer** – any change to serialization that affects downstream consumers must bump + the exporter version string and update `CANONICAL_RECORDS.md`. +- **Schema-first** – update documentation (`CANONICAL_RECORDS.md`) and corresponding tests before + shipping new fields. +- **Dual-read period** – when introducing a new field, keep old readers working by: + 1. Making the field optional in the canonical model. + 2. Providing default behavior in exporters/mergers when the field is absent. + 3. Communicating via release notes and toggles when the field will become required. + +## Workflow for Changes + +1. **Proposal** – raise an issue describing the motivation, affected records, and compatibility + impact. Link to the relevant task in `TASKS.md`. +2. **Docs + Tests first** – update `CANONICAL_RECORDS.md`, add/adjust golden fixtures, and extend + regression tests (hash comparisons, snapshot assertions) to capture the new shape. +3. **Implementation** – introduce the model change along with migration logic (e.g., mergers filling + defaults, exporters emitting the new payload). +4. **Exporter bump** – update exporter version manifests (`ExporterVersion.GetVersion`) whenever the + serialized payload differs. +5. **Announcement** – document the change in release notes, highlighting optional vs. required + timelines. +6. **Cleanup** – once consumers have migrated, remove transitional logic and update docs/tests to + reflect the permanent shape. + +## Testing Checklist + +- `StellaOps.Feedser.Models.Tests` – update unit tests and golden examples. +- `Serialization determinism` – ensure the hash regression tests cover the new fields. +- Exporter integration (`Json`, `TrivyDb`) – confirm manifests include provenance + tree metadata + for the new shape. + +Following this playbook keeps canonical payloads stable while allowing incremental evolution. diff --git a/src/StellaOps.Feedser.Models/CANONICAL_RECORDS.md b/src/StellaOps.Feedser.Models/CANONICAL_RECORDS.md new file mode 100644 index 00000000..0eaf32f4 --- /dev/null +++ b/src/StellaOps.Feedser.Models/CANONICAL_RECORDS.md @@ -0,0 +1,144 @@ +# Canonical Record Definitions + +> Source of truth for the normalized advisory schema emitted by `StellaOps.Feedser.Models`. +> Keep this document in sync with the public record types under `StellaOps.Feedser.Models` and +> update it whenever a new field is introduced or semantics change. + +## Advisory + +| Field | Type | Required | Notes | +|-------|------|----------|-------| +| `advisoryKey` | string | yes | Globally unique identifier selected by the merge layer (often a CVE/GHSA/vendor key). Stored lowercased unless vendor casing is significant. | +| `title` | string | yes | Human readable title. Must be non-empty and trimmed. | +| `summary` | string? | optional | Short description; trimmed to `null` when empty. | +| `language` | string? | optional | ISO language code (lowercase). | +| `published` | DateTimeOffset? | optional | UTC timestamp when vendor originally published. | +| `modified` | DateTimeOffset? | optional | UTC timestamp when vendor last updated. | +| `severity` | string? | optional | Normalized severity label (`critical`, `high`, etc.). | +| `exploitKnown` | bool | yes | Whether KEV/other sources confirm active exploitation. | +| `aliases` | string[] | yes | Sorted, de-duplicated list of normalized aliases (see [Alias Schemes](#alias-schemes)). | +| `credits` | AdvisoryCredit[] | yes | Deterministically ordered acknowledgements (role + contact metadata). | +| `references` | AdvisoryReference[] | yes | Deterministically ordered reference set. | +| `affectedPackages` | AffectedPackage[] | yes | Deterministically ordered affected packages. | +| `cvssMetrics` | CvssMetric[] | yes | Deterministically ordered CVSS metrics (v3, v4 first). | +| `provenance` | AdvisoryProvenance[] | yes | Normalized provenance entries sorted by source then kind then recorded timestamp. | + +### Invariants +- Collections are immutable (`ImmutableArray`) and always sorted deterministically. +- `AdvisoryKey` and `Title` are mandatory and trimmed. +- All timestamps are stored as UTC. +- Aliases and references leverage helper registries for validation. + +## AdvisoryReference + +| Field | Type | Required | Notes | +|-------|------|----------|-------| +| `url` | string | yes | Absolute HTTP/HTTPS URL. | +| `kind` | string? | optional | Categorized reference role (e.g. `advisory`, `patch`, `changelog`). | +| `sourceTag` | string? | optional | Free-form tag identifying originating source. | +| `summary` | string? | optional | Short description. | +| `provenance` | AdvisoryProvenance | yes | Provenance entry describing how the reference was mapped. | + +Deterministic ordering: by `url`, then `kind`, then `sourceTag`, then `provenance.RecordedAt`. + +## AdvisoryCredit + +| Field | Type | Required | Notes | +|-------|------|----------|-------| +| `displayName` | string | yes | Human-readable acknowledgement (reporter, maintainer, analyst, etc.). | +| `role` | string? | optional | Normalized role token (lowercase with `_` separators). | +| `contacts` | string[] | yes | Sorted set of vendor-supplied handles or URLs; may be empty. | +| `provenance` | AdvisoryProvenance | yes | Provenance entry describing how the credit was captured. | + +Deterministic ordering: by `role` (nulls first) then `displayName`. + +## AffectedPackage + +| Field | Type | Required | Notes | +|-------|------|----------|-------| +| `type` | string | yes | Semantic type (`semver`, `rpm`, `deb`, `purl`, `cpe`, etc.). Lowercase. | +| `identifier` | string | yes | Canonical identifier (package name, PURL, CPE, NEVRA, etc.). | +| `platform` | string? | optional | Explicit platform / distro (e.g. `ubuntu`, `rhel-8`). | +| `versionRanges` | AffectedVersionRange[] | yes | Deduplicated + sorted by introduced/fixed/last/expr/kind. | +| `statuses` | AffectedPackageStatus[] | yes | Optional status flags (e.g. `fixed`, `affected`). | +| `provenance` | AdvisoryProvenance[] | yes | Provenance entries for package level metadata. | + +Deterministic ordering: packages sorted by `type`, then `identifier`, then `platform` (ordinal). + +## AffectedVersionRange + +| Field | Type | Required | Notes | +|-------|------|----------|-------| +| `rangeKind` | string | yes | Classification of range semantics (`semver`, `evr`, `nevra`, `version`, `purl`). Lowercase. | +| `introducedVersion` | string? | optional | Inclusive lower bound when impact begins. | +| `fixedVersion` | string? | optional | Exclusive bounding version containing the fix. | +| `lastAffectedVersion` | string? | optional | Inclusive upper bound when no fix exists. | +| `rangeExpression` | string? | optional | Normalized textual expression for non-simple ranges. | +| `provenance` | AdvisoryProvenance | yes | Provenance entry for the range. | +| `primitives` | RangePrimitives? | optional | Structured metadata (SemVer/Nevra/Evr/vendor extensions) when available. | + +Comparers/equality ignore provenance differences. + +## CvssMetric + +| Field | Type | Required | Notes | +|-------|------|----------|-------| +| `version` | string | yes | `2.0`, `3.0`, `3.1`, `4.0`, etc. | +| `vector` | string | yes | Official CVSS vector string. | +| `score` | double | yes | CVSS base score (0.0-10.0). | +| `severity` | string | yes | Severity label mapped from score or vendor metadata. | +| `provenance` | AdvisoryProvenance | yes | Provenance entry. | + +Sorted by version then vector for determinism. + +## AdvisoryProvenance + +| Field | Type | Required | Notes | +|-------|------|----------|-------| +| `source` | string | yes | Logical source identifier (`nvd`, `redhat`, `osv`, etc.). | +| `kind` | string | yes | Operation performed (`fetch`, `parse`, `map`, `merge`, `enrich`). | +| `value` | string? | optional | Free-form pipeline detail (parser identifier, rule set, resume cursor). | +| `recordedAt` | DateTimeOffset | yes | UTC timestamp when provenance was captured. | +| `fieldMask` | string[] | optional | Canonical field coverage expressed as lowercase masks (e.g. `affectedpackages[]`, `affectedpackages[].versionranges[]`). | + +### Provenance Mask Expectations +Each canonical field is expected to carry at least one provenance entry derived from the +responsible pipeline stage. Populate `fieldMask` with the lowercase canonical mask(s) describing the +covered field(s); downstream metrics and resume helpers rely on this signal to reason about +coverage. When aggregating provenance from subcomponents (e.g., affected package ranges), merge code +should ensure: + +- Advisory level provenance documents the source document and merge actions. +- References, packages, ranges, and metrics each include their own provenance entry reflecting + the most specific source (vendor feed, computed normalization, etc.). +- Export-specific metadata (digest manifests, offline bundles) include exporter version alongside + the builder metadata. + +## Alias Schemes + +Supported alias scheme prefixes: + +- `CVE-` +- `GHSA-` +- `OSV-` +- `JVN-`, `JVNDB-` +- `BDU-` +- `VU#` +- `MSRC-` +- `CISCO-SA-` +- `ORACLE-CPU` +- `APSB-`, `APA-` +- `APPLE-HT` +- `CHROMIUM:` / `CHROMIUM-` +- `VMSA-` +- `RHSA-` +- `USN-` +- `DSA-` +- `SUSE-SU-` +- `ICSA-` +- `CWE-` +- `cpe:` +- `pkg:` (Package URL / PURL) + +The registry exposed via `AliasSchemes` and `AliasSchemeRegistry` can be used to validate aliases and +drive downstream conditionals without re-implementing pattern rules. diff --git a/src/StellaOps.Feedser.Models/CanonicalJsonSerializer.cs b/src/StellaOps.Feedser.Models/CanonicalJsonSerializer.cs new file mode 100644 index 00000000..e6d45a2c --- /dev/null +++ b/src/StellaOps.Feedser.Models/CanonicalJsonSerializer.cs @@ -0,0 +1,161 @@ +using System.Collections.Generic; +using System.Linq; +using System.Text.Encodings.Web; +using System.Text.Json; +using System.Text.Json.Serialization; +using System.Text.Json.Serialization.Metadata; + +namespace StellaOps.Feedser.Models; + +/// +/// Deterministic JSON serializer tuned for canonical advisory output. +/// +public static class CanonicalJsonSerializer +{ + private static readonly JsonSerializerOptions CompactOptions = CreateOptions(writeIndented: false); + private static readonly JsonSerializerOptions PrettyOptions = CreateOptions(writeIndented: true); + + private static readonly IReadOnlyDictionary PropertyOrderOverrides = new Dictionary + { + { + typeof(AdvisoryProvenance), + new[] + { + "source", + "kind", + "value", + "decisionReason", + "recordedAt", + "fieldMask", + } + }, + { + typeof(AffectedPackage), + new[] + { + "type", + "identifier", + "platform", + "versionRanges", + "normalizedVersions", + "statuses", + "provenance", + } + }, + { + typeof(AdvisoryCredit), + new[] + { + "displayName", + "role", + "contacts", + "provenance", + } + }, + { + typeof(NormalizedVersionRule), + new[] + { + "scheme", + "type", + "min", + "minInclusive", + "max", + "maxInclusive", + "value", + "notes", + } + }, + }; + + public static string Serialize(T value) + => JsonSerializer.Serialize(value, CompactOptions); + + public static string SerializeIndented(T value) + => JsonSerializer.Serialize(value, PrettyOptions); + + public static Advisory Normalize(Advisory advisory) + => new( + advisory.AdvisoryKey, + advisory.Title, + advisory.Summary, + advisory.Language, + advisory.Published, + advisory.Modified, + advisory.Severity, + advisory.ExploitKnown, + advisory.Aliases, + advisory.Credits, + advisory.References, + advisory.AffectedPackages, + advisory.CvssMetrics, + advisory.Provenance); + + public static T Deserialize(string json) + => JsonSerializer.Deserialize(json, PrettyOptions)! + ?? throw new InvalidOperationException($"Unable to deserialize type {typeof(T).Name}."); + + private static JsonSerializerOptions CreateOptions(bool writeIndented) + { + var options = new JsonSerializerOptions + { + PropertyNamingPolicy = JsonNamingPolicy.CamelCase, + DictionaryKeyPolicy = JsonNamingPolicy.CamelCase, + DefaultIgnoreCondition = JsonIgnoreCondition.Never, + WriteIndented = writeIndented, + Encoder = JavaScriptEncoder.UnsafeRelaxedJsonEscaping, + }; + + var baselineResolver = options.TypeInfoResolver ?? new DefaultJsonTypeInfoResolver(); + options.TypeInfoResolver = new DeterministicTypeInfoResolver(baselineResolver); + options.Converters.Add(new JsonStringEnumConverter(JsonNamingPolicy.CamelCase, allowIntegerValues: false)); + return options; + } + + private sealed class DeterministicTypeInfoResolver : IJsonTypeInfoResolver + { + private readonly IJsonTypeInfoResolver _inner; + + public DeterministicTypeInfoResolver(IJsonTypeInfoResolver inner) + { + _inner = inner ?? throw new ArgumentNullException(nameof(inner)); + } + + public JsonTypeInfo GetTypeInfo(Type type, JsonSerializerOptions options) + { + var info = _inner.GetTypeInfo(type, options); + if (info is null) + { + throw new InvalidOperationException($"Unable to resolve JsonTypeInfo for '{type}'."); + } + + if (info.Kind is JsonTypeInfoKind.Object && info.Properties is { Count: > 1 }) + { + var ordered = info.Properties + .OrderBy(property => GetPropertyOrder(type, property.Name)) + .ThenBy(property => property.Name, StringComparer.Ordinal) + .ToArray(); + + info.Properties.Clear(); + foreach (var property in ordered) + { + info.Properties.Add(property); + } + } + + return info; + } + + private static int GetPropertyOrder(Type type, string propertyName) + { + if (PropertyOrderOverrides.TryGetValue(type, out var order) && + Array.IndexOf(order, propertyName) is var index && + index >= 0) + { + return index; + } + + return int.MaxValue; + } + } +} diff --git a/src/StellaOps.Feedser.Models/CvssMetric.cs b/src/StellaOps.Feedser.Models/CvssMetric.cs new file mode 100644 index 00000000..90765754 --- /dev/null +++ b/src/StellaOps.Feedser.Models/CvssMetric.cs @@ -0,0 +1,31 @@ +using System.Text.Json.Serialization; + +namespace StellaOps.Feedser.Models; + +/// +/// Canonicalized CVSS metric details supporting deterministic serialization. +/// +public sealed record CvssMetric +{ + public static CvssMetric Empty { get; } = new("3.1", vector: "CVSS:3.1/AV:N/AC:L/PR:N/UI:N/S:U/C:N/I:N/A:N", baseScore: 0, baseSeverity: "none", provenance: AdvisoryProvenance.Empty); + + [JsonConstructor] + public CvssMetric(string version, string vector, double baseScore, string baseSeverity, AdvisoryProvenance provenance) + { + Version = Validation.EnsureNotNullOrWhiteSpace(version, nameof(version)); + Vector = Validation.EnsureNotNullOrWhiteSpace(vector, nameof(vector)); + BaseSeverity = Validation.EnsureNotNullOrWhiteSpace(baseSeverity, nameof(baseSeverity)).ToLowerInvariant(); + BaseScore = Math.Round(baseScore, 1, MidpointRounding.AwayFromZero); + Provenance = provenance ?? AdvisoryProvenance.Empty; + } + + public string Version { get; } + + public string Vector { get; } + + public double BaseScore { get; } + + public string BaseSeverity { get; } + + public AdvisoryProvenance Provenance { get; } +} diff --git a/src/StellaOps.Feedser.Models/NormalizedVersionRule.cs b/src/StellaOps.Feedser.Models/NormalizedVersionRule.cs new file mode 100644 index 00000000..86406b34 --- /dev/null +++ b/src/StellaOps.Feedser.Models/NormalizedVersionRule.cs @@ -0,0 +1,185 @@ +using System; +using System.Collections.Generic; + +namespace StellaOps.Feedser.Models; + +/// +/// Canonical normalized version rule emitted by range builders for analytical queries. +/// +public sealed record NormalizedVersionRule +{ + public NormalizedVersionRule( + string scheme, + string type, + string? min = null, + bool? minInclusive = null, + string? max = null, + bool? maxInclusive = null, + string? value = null, + string? notes = null) + { + Scheme = Validation.EnsureNotNullOrWhiteSpace(scheme, nameof(scheme)).ToLowerInvariant(); + Type = Validation.EnsureNotNullOrWhiteSpace(type, nameof(type)).Replace('_', '-').ToLowerInvariant(); + Min = Validation.TrimToNull(min); + MinInclusive = minInclusive; + Max = Validation.TrimToNull(max); + MaxInclusive = maxInclusive; + Value = Validation.TrimToNull(value); + Notes = Validation.TrimToNull(notes); + } + + public string Scheme { get; } + + public string Type { get; } + + public string? Min { get; } + + public bool? MinInclusive { get; } + + public string? Max { get; } + + public bool? MaxInclusive { get; } + + public string? Value { get; } + + public string? Notes { get; } +} + +public sealed class NormalizedVersionRuleEqualityComparer : IEqualityComparer +{ + public static NormalizedVersionRuleEqualityComparer Instance { get; } = new(); + + public bool Equals(NormalizedVersionRule? x, NormalizedVersionRule? y) + { + if (ReferenceEquals(x, y)) + { + return true; + } + + if (x is null || y is null) + { + return false; + } + + return string.Equals(x.Scheme, y.Scheme, StringComparison.Ordinal) + && string.Equals(x.Type, y.Type, StringComparison.Ordinal) + && string.Equals(x.Min, y.Min, StringComparison.Ordinal) + && x.MinInclusive == y.MinInclusive + && string.Equals(x.Max, y.Max, StringComparison.Ordinal) + && x.MaxInclusive == y.MaxInclusive + && string.Equals(x.Value, y.Value, StringComparison.Ordinal) + && string.Equals(x.Notes, y.Notes, StringComparison.Ordinal); + } + + public int GetHashCode(NormalizedVersionRule obj) + => HashCode.Combine( + obj.Scheme, + obj.Type, + obj.Min, + obj.MinInclusive, + obj.Max, + obj.MaxInclusive, + obj.Value, + obj.Notes); +} + +public sealed class NormalizedVersionRuleComparer : IComparer +{ + public static NormalizedVersionRuleComparer Instance { get; } = new(); + + public int Compare(NormalizedVersionRule? x, NormalizedVersionRule? y) + { + if (ReferenceEquals(x, y)) + { + return 0; + } + + if (x is null) + { + return -1; + } + + if (y is null) + { + return 1; + } + + var schemeComparison = string.Compare(x.Scheme, y.Scheme, StringComparison.Ordinal); + if (schemeComparison != 0) + { + return schemeComparison; + } + + var typeComparison = string.Compare(x.Type, y.Type, StringComparison.Ordinal); + if (typeComparison != 0) + { + return typeComparison; + } + + var minComparison = string.Compare(x.Min, y.Min, StringComparison.Ordinal); + if (minComparison != 0) + { + return minComparison; + } + + var minInclusiveComparison = NullableBoolCompare(x.MinInclusive, y.MinInclusive); + if (minInclusiveComparison != 0) + { + return minInclusiveComparison; + } + + var maxComparison = string.Compare(x.Max, y.Max, StringComparison.Ordinal); + if (maxComparison != 0) + { + return maxComparison; + } + + var maxInclusiveComparison = NullableBoolCompare(x.MaxInclusive, y.MaxInclusive); + if (maxInclusiveComparison != 0) + { + return maxInclusiveComparison; + } + + var valueComparison = string.Compare(x.Value, y.Value, StringComparison.Ordinal); + if (valueComparison != 0) + { + return valueComparison; + } + + return string.Compare(x.Notes, y.Notes, StringComparison.Ordinal); + } + + private static int NullableBoolCompare(bool? x, bool? y) + { + if (x == y) + { + return 0; + } + + return (x, y) switch + { + (null, not null) => -1, + (not null, null) => 1, + (false, true) => -1, + (true, false) => 1, + _ => 0, + }; + } +} + +public static class NormalizedVersionSchemes +{ + public const string SemVer = "semver"; + public const string Nevra = "nevra"; + public const string Evr = "evr"; +} + +public static class NormalizedVersionRuleTypes +{ + public const string Range = "range"; + public const string Exact = "exact"; + public const string LessThan = "lt"; + public const string LessThanOrEqual = "lte"; + public const string GreaterThan = "gt"; + public const string GreaterThanOrEqual = "gte"; +} diff --git a/src/StellaOps.Feedser.Models/OsvGhsaParityDiagnostics.cs b/src/StellaOps.Feedser.Models/OsvGhsaParityDiagnostics.cs new file mode 100644 index 00000000..fa2757b0 --- /dev/null +++ b/src/StellaOps.Feedser.Models/OsvGhsaParityDiagnostics.cs @@ -0,0 +1,72 @@ +using System; +using System.Collections.Generic; +using System.Diagnostics.Metrics; + +namespace StellaOps.Feedser.Models; + +/// +/// Emits telemetry for OSV vs GHSA parity reports so QA dashboards can track regression trends. +/// +public static class OsvGhsaParityDiagnostics +{ + private static readonly Meter Meter = new("StellaOps.Feedser.Models.OsvGhsaParity"); + private static readonly Counter TotalCounter = Meter.CreateCounter( + "feedser.osv_ghsa.total", + unit: "count", + description: "Total GHSA identifiers evaluated for OSV parity."); + private static readonly Counter IssueCounter = Meter.CreateCounter( + "feedser.osv_ghsa.issues", + unit: "count", + description: "Parity issues grouped by dataset, issue kind, and field mask."); + + public static void RecordReport(OsvGhsaParityReport report, string dataset) + { + ArgumentNullException.ThrowIfNull(report); + dataset = NormalizeDataset(dataset); + + if (report.TotalGhsaIds > 0) + { + TotalCounter.Add(report.TotalGhsaIds, CreateTotalTags(dataset)); + } + + if (!report.HasIssues) + { + return; + } + + foreach (var issue in report.Issues) + { + IssueCounter.Add(1, CreateIssueTags(dataset, issue)); + } + } + + private static KeyValuePair[] CreateTotalTags(string dataset) + => new[] + { + new KeyValuePair("dataset", dataset), + }; + + private static KeyValuePair[] CreateIssueTags(string dataset, OsvGhsaParityIssue issue) + { + var mask = issue.FieldMask.IsDefaultOrEmpty + ? "none" + : string.Join('|', issue.FieldMask); + + return new[] + { + new KeyValuePair("dataset", dataset), + new KeyValuePair("issueKind", issue.IssueKind), + new KeyValuePair("fieldMask", mask), + }; + } + + private static string NormalizeDataset(string dataset) + { + if (string.IsNullOrWhiteSpace(dataset)) + { + return "default"; + } + + return dataset.Trim().ToLowerInvariant(); + } +} diff --git a/src/StellaOps.Feedser.Models/OsvGhsaParityInspector.cs b/src/StellaOps.Feedser.Models/OsvGhsaParityInspector.cs new file mode 100644 index 00000000..0ab7e862 --- /dev/null +++ b/src/StellaOps.Feedser.Models/OsvGhsaParityInspector.cs @@ -0,0 +1,183 @@ +using System; +using System.Collections.Generic; +using System.Collections.Immutable; +using System.Linq; + +namespace StellaOps.Feedser.Models; + +/// +/// Compares OSV and GHSA advisory datasets to surface mismatches in coverage, severity, or presence. +/// +public static class OsvGhsaParityInspector +{ + public static OsvGhsaParityReport Compare(IEnumerable osvAdvisories, IEnumerable ghsaAdvisories) + { + ArgumentNullException.ThrowIfNull(osvAdvisories); + ArgumentNullException.ThrowIfNull(ghsaAdvisories); + + var osvByGhsa = BuildOsvMap(osvAdvisories); + var ghsaById = BuildGhsaMap(ghsaAdvisories); + + var union = osvByGhsa.Keys + .Union(ghsaById.Keys, StringComparer.OrdinalIgnoreCase) + .OrderBy(static key => key, StringComparer.OrdinalIgnoreCase) + .ToArray(); + + var issues = ImmutableArray.CreateBuilder(); + + foreach (var ghsaId in union) + { + osvByGhsa.TryGetValue(ghsaId, out var osv); + ghsaById.TryGetValue(ghsaId, out var ghsa); + var normalizedId = ghsaId.ToUpperInvariant(); + + if (osv is null) + { + issues.Add(new OsvGhsaParityIssue( + normalizedId, + "missing_osv", + "GHSA advisory missing from OSV dataset.", + ImmutableArray.Create(ProvenanceFieldMasks.AffectedPackages))); + continue; + } + + if (ghsa is null) + { + issues.Add(new OsvGhsaParityIssue( + normalizedId, + "missing_ghsa", + "OSV mapped GHSA alias without a matching GHSA advisory.", + ImmutableArray.Create(ProvenanceFieldMasks.AffectedPackages))); + continue; + } + + if (!SeverityMatches(osv, ghsa)) + { + var detail = $"Severity mismatch: OSV={osv.Severity ?? "(null)"}, GHSA={ghsa.Severity ?? "(null)"}."; + issues.Add(new OsvGhsaParityIssue( + normalizedId, + "severity_mismatch", + detail, + ImmutableArray.Create(ProvenanceFieldMasks.Advisory))); + } + + if (!RangeCoverageMatches(osv, ghsa)) + { + var detail = $"Range coverage mismatch: OSV ranges={CountRanges(osv)}, GHSA ranges={CountRanges(ghsa)}."; + issues.Add(new OsvGhsaParityIssue( + normalizedId, + "range_mismatch", + detail, + ImmutableArray.Create(ProvenanceFieldMasks.VersionRanges))); + } + } + + return new OsvGhsaParityReport(union.Length, issues.ToImmutable()); + } + + private static IReadOnlyDictionary BuildOsvMap(IEnumerable advisories) + { + var comparer = StringComparer.OrdinalIgnoreCase; + var map = new Dictionary(comparer); + + foreach (var advisory in advisories) + { + if (advisory is null) + { + continue; + } + + foreach (var alias in advisory.Aliases) + { + if (alias.StartsWith("ghsa-", StringComparison.OrdinalIgnoreCase)) + { + map.TryAdd(alias, advisory); + } + } + } + + return map; + } + + private static IReadOnlyDictionary BuildGhsaMap(IEnumerable advisories) + { + var comparer = StringComparer.OrdinalIgnoreCase; + var map = new Dictionary(comparer); + + foreach (var advisory in advisories) + { + if (advisory is null) + { + continue; + } + + if (advisory.AdvisoryKey.StartsWith("ghsa-", StringComparison.OrdinalIgnoreCase)) + { + map.TryAdd(advisory.AdvisoryKey, advisory); + continue; + } + + foreach (var alias in advisory.Aliases) + { + if (alias.StartsWith("ghsa-", StringComparison.OrdinalIgnoreCase)) + { + map.TryAdd(alias, advisory); + } + } + } + + return map; + } + + private static bool SeverityMatches(Advisory osv, Advisory ghsa) + => string.Equals(osv.Severity, ghsa.Severity, StringComparison.OrdinalIgnoreCase); + + private static bool RangeCoverageMatches(Advisory osv, Advisory ghsa) + { + var osvRanges = CountRanges(osv); + var ghsaRanges = CountRanges(ghsa); + if (osvRanges == ghsaRanges) + { + return true; + } + + // Consider zero-vs-nonzero mismatches as actionable even if raw counts differ. + return osvRanges == 0 && ghsaRanges == 0; + } + + private static int CountRanges(Advisory advisory) + { + if (advisory.AffectedPackages.IsDefaultOrEmpty) + { + return 0; + } + + var count = 0; + foreach (var package in advisory.AffectedPackages) + { + if (package.VersionRanges.IsDefaultOrEmpty) + { + continue; + } + + count += package.VersionRanges.Length; + } + + return count; + } +} + +public sealed record OsvGhsaParityIssue( + string GhsaId, + string IssueKind, + string Detail, + ImmutableArray FieldMask); + +public sealed record OsvGhsaParityReport(int TotalGhsaIds, ImmutableArray Issues) +{ + public bool HasIssues => !Issues.IsDefaultOrEmpty && Issues.Length > 0; + + public int MissingFromOsv => Issues.Count(issue => issue.IssueKind.Equals("missing_osv", StringComparison.OrdinalIgnoreCase)); + + public int MissingFromGhsa => Issues.Count(issue => issue.IssueKind.Equals("missing_ghsa", StringComparison.OrdinalIgnoreCase)); +} diff --git a/src/StellaOps.Feedser.Models/PROVENANCE_GUIDELINES.md b/src/StellaOps.Feedser.Models/PROVENANCE_GUIDELINES.md new file mode 100644 index 00000000..4c6f9166 --- /dev/null +++ b/src/StellaOps.Feedser.Models/PROVENANCE_GUIDELINES.md @@ -0,0 +1,15 @@ +# Canonical Field Provenance Guidelines + +- **Always attach provenance** when mapping any field into `StellaOps.Feedser.Models`. Use `AdvisoryProvenance` to capture `source` (feed identifier), `kind` (fetch|parse|map|merge), `value` (cursor or extractor hint), and the UTC timestamp when it was recorded. +- **Per-field strategy** + - `Advisory` metadata (title, summary, severity) should record the connector responsible for the value. When merge overrides occur, add an additional provenance record rather than mutating the original. + - `References` must record whether the link originated from the primary advisory (`kind=advisory`), a vendor patch (`kind=patch`), or an enrichment feed (`kind=enrichment`). + - `AffectedPackage` records should capture the exact extraction routine (e.g., `map:oval`, `map:nvd`, `map:vendor`). + - `CvssMetric` provenance should include the scoring authority (e.g., `nvd`, `redhat`) and whether it was supplied or derived. + - `AffectedVersionRange` provenance anchors the transcript used to build the range. Preserve version strings as given by the source to aid debugging. +- **Merge policy**: never discard provenance when merging; instead append a new `AdvisoryProvenance` entry with the merge routine (`source=merge.determine-precedence`). +- **Determinism**: provenance collections are sorted by source → kind → recordedAt before serialization; avoid generating random identifiers inside provenance. +- **Field masks**: populate `fieldMask` on each provenance entry using lowercase canonical masks (see `ProvenanceFieldMasks`). This powers metrics, parity checks, and resume diagnostics. Recent additions include `affectedpackages[].normalizedversions[]`, `affectedpackages[].versionranges[].primitives.semver`, and `credits[]`. +- **Redaction**: keep provenance values free of secrets; prefer tokens or normalized descriptors when referencing authenticated fetches. +- **Range telemetry**: each `AffectedVersionRange` is observed by the `feedser.range.primitives` metric. Emit the richest `RangePrimitives` possible (SemVer/NEVRA/EVR plus vendor extensions); the telemetry tags make it easy to spot connectors missing structured range data. +- **Vendor extensions**: when vendor feeds surface bespoke status flags, capture them in `RangePrimitives.VendorExtensions`. SUSE advisories publish `suse.status` (open/resolved/investigating) and Ubuntu notices expose `ubuntu.pocket`/`ubuntu.release` to distinguish security vs ESM pockets; Adobe APSB bulletins emit `adobe.track`, `adobe.platform`, `adobe.priority`, `adobe.availability`, plus `adobe.affected.raw`/`adobe.updated.raw` to preserve PSIRT metadata while keeping the status catalog canonical. These values are exported for dashboards and alerting. diff --git a/src/StellaOps.Feedser.Models/ProvenanceFieldMasks.cs b/src/StellaOps.Feedser.Models/ProvenanceFieldMasks.cs new file mode 100644 index 00000000..101ef46a --- /dev/null +++ b/src/StellaOps.Feedser.Models/ProvenanceFieldMasks.cs @@ -0,0 +1,16 @@ +namespace StellaOps.Feedser.Models; + +/// +/// Canonical field-mask identifiers for provenance coverage. +/// +public static class ProvenanceFieldMasks +{ + public const string Advisory = "advisory"; + public const string References = "references[]"; + public const string Credits = "credits[]"; + public const string AffectedPackages = "affectedpackages[]"; + public const string VersionRanges = "affectedpackages[].versionranges[]"; + public const string NormalizedVersions = "affectedpackages[].normalizedversions[]"; + public const string PackageStatuses = "affectedpackages[].statuses[]"; + public const string CvssMetrics = "cvssmetrics[]"; +} diff --git a/src/StellaOps.Feedser.Models/ProvenanceInspector.cs b/src/StellaOps.Feedser.Models/ProvenanceInspector.cs new file mode 100644 index 00000000..d867e14b --- /dev/null +++ b/src/StellaOps.Feedser.Models/ProvenanceInspector.cs @@ -0,0 +1,297 @@ +using System; +using System.Collections.Generic; +using System.Collections.Immutable; +using System.Diagnostics.Metrics; +using System.Linq; +using Microsoft.Extensions.Logging; + +namespace StellaOps.Feedser.Models; + +public static class ProvenanceInspector +{ + public static IReadOnlyList FindMissingProvenance(Advisory advisory) + { + var results = new List(); + var source = advisory.Provenance.FirstOrDefault()?.Source ?? "unknown"; + + if (advisory.Provenance.Length == 0) + { + results.Add(new MissingProvenance(source, "advisory", null, ImmutableArray.Create(ProvenanceFieldMasks.Advisory))); + } + + foreach (var reference in advisory.References) + { + if (IsMissing(reference.Provenance)) + { + results.Add(new MissingProvenance( + reference.Provenance.Source ?? source, + $"reference:{reference.Url}", + reference.Provenance.RecordedAt, + NormalizeMask(reference.Provenance.FieldMask, ProvenanceFieldMasks.References))); + } + } + + foreach (var package in advisory.AffectedPackages) + { + if (package.Provenance.Length == 0) + { + results.Add(new MissingProvenance( + source, + $"package:{package.Identifier}", + null, + ImmutableArray.Create(ProvenanceFieldMasks.AffectedPackages))); + } + + foreach (var range in package.VersionRanges) + { + ProvenanceDiagnostics.RecordRangePrimitive(range.Provenance.Source ?? source, range); + + if (IsMissing(range.Provenance)) + { + results.Add(new MissingProvenance( + range.Provenance.Source ?? source, + $"range:{package.Identifier}", + range.Provenance.RecordedAt, + NormalizeMask(range.Provenance.FieldMask, ProvenanceFieldMasks.VersionRanges))); + } + } + + foreach (var status in package.Statuses) + { + if (IsMissing(status.Provenance)) + { + results.Add(new MissingProvenance( + status.Provenance.Source ?? source, + $"status:{package.Identifier}:{status.Status}", + status.Provenance.RecordedAt, + NormalizeMask(status.Provenance.FieldMask, ProvenanceFieldMasks.PackageStatuses))); + } + } + } + + foreach (var metric in advisory.CvssMetrics) + { + if (IsMissing(metric.Provenance)) + { + results.Add(new MissingProvenance( + metric.Provenance.Source ?? source, + $"cvss:{metric.Version}", + metric.Provenance.RecordedAt, + NormalizeMask(metric.Provenance.FieldMask, ProvenanceFieldMasks.CvssMetrics))); + } + } + + return results; + } + + private static bool IsMissing(AdvisoryProvenance provenance) + { + return provenance == AdvisoryProvenance.Empty + || string.IsNullOrWhiteSpace(provenance.Source) + || string.IsNullOrWhiteSpace(provenance.Kind); + } + + private static ImmutableArray NormalizeMask(ImmutableArray mask, string fallback) + { + if (mask.IsDefaultOrEmpty) + { + return ImmutableArray.Create(fallback); + } + + return mask; + } + +} + +public sealed record MissingProvenance( + string Source, + string Component, + DateTimeOffset? RecordedAt, + ImmutableArray FieldMask); + +public static class ProvenanceDiagnostics +{ + private static readonly Meter Meter = new("StellaOps.Feedser.Models.Provenance"); + private static readonly Counter MissingCounter = Meter.CreateCounter( + "feedser.provenance.missing", + unit: "count", + description: "Number of canonical objects missing provenance metadata."); + private static readonly Counter RangePrimitiveCounter = Meter.CreateCounter( + "feedser.range.primitives", + unit: "count", + description: "Range coverage by kind, primitive availability, and vendor extensions."); + + private static readonly object SyncRoot = new(); + private static readonly Dictionary EarliestMissing = new(StringComparer.OrdinalIgnoreCase); + private static readonly HashSet RecordedComponents = new(StringComparer.OrdinalIgnoreCase); + + public static void RecordMissing( + string source, + string component, + DateTimeOffset? recordedAt, + IReadOnlyList? fieldMask = null) + { + if (string.IsNullOrWhiteSpace(source)) + { + source = "unknown"; + } + + component = string.IsNullOrWhiteSpace(component) ? "unknown" : component.Trim(); + var maskKey = NormalizeMask(fieldMask); + + bool shouldRecord; + lock (SyncRoot) + { + var key = $"{source}|{component}|{maskKey}"; + shouldRecord = RecordedComponents.Add(key); + + if (recordedAt.HasValue) + { + if (!EarliestMissing.TryGetValue(source, out var existing) || recordedAt.Value < existing) + { + EarliestMissing[source] = recordedAt.Value; + } + } + } + + if (!shouldRecord) + { + return; + } + + var category = DetermineCategory(component); + var severity = DetermineSeverity(category); + + var tags = new[] + { + new KeyValuePair("source", source), + new KeyValuePair("component", component), + new KeyValuePair("category", category), + new KeyValuePair("severity", severity), + new KeyValuePair("fieldMask", string.IsNullOrEmpty(maskKey) ? "none" : maskKey), + }; + MissingCounter.Add(1, tags); + } + + public static void ReportResumeWindow(string source, DateTimeOffset windowStart, ILogger logger) + { + if (string.IsNullOrWhiteSpace(source) || logger is null) + { + return; + } + + DateTimeOffset earliest; + var hasEntry = false; + lock (SyncRoot) + { + if (EarliestMissing.TryGetValue(source, out earliest)) + { + hasEntry = true; + if (windowStart <= earliest) + { + EarliestMissing.Remove(source); + var prefix = source + "|"; + RecordedComponents.RemoveWhere(entry => entry.StartsWith(prefix, StringComparison.OrdinalIgnoreCase)); + } + } + } + + if (!hasEntry) + { + return; + } + + if (windowStart <= earliest) + { + logger.LogInformation( + "Resume window starting {WindowStart:o} for {Source} may backfill missing provenance recorded at {Earliest:o}.", + windowStart, + source, + earliest); + } + else + { + logger.LogInformation( + "Earliest missing provenance for {Source} remains at {Earliest:o}; current resume window begins at {WindowStart:o}. Consider widening overlap to backfill.", + source, + earliest, + windowStart); + } + } + + public static void RecordRangePrimitive(string source, AffectedVersionRange range) + { + if (range is null) + { + return; + } + + source = string.IsNullOrWhiteSpace(source) ? "unknown" : source.Trim(); + + var primitives = range.Primitives; + var primitiveKinds = DeterminePrimitiveKinds(primitives); + var vendorExtensions = primitives?.VendorExtensions?.Count ?? 0; + + var tags = new[] + { + new KeyValuePair("source", source), + new KeyValuePair("rangeKind", string.IsNullOrWhiteSpace(range.RangeKind) ? "unknown" : range.RangeKind), + new KeyValuePair("primitiveKinds", primitiveKinds), + new KeyValuePair("hasVendorExtensions", vendorExtensions > 0 ? "true" : "false"), + }; + + RangePrimitiveCounter.Add(1, tags); + } + + private static string DetermineCategory(string component) + { + if (string.IsNullOrWhiteSpace(component)) + { + return "unknown"; + } + + var index = component.IndexOf(':'); + var category = index > 0 ? component[..index] : component; + return category.Trim().ToLowerInvariant(); + } + + private static string DetermineSeverity(string category) + => category switch + { + "advisory" => "critical", + "package" => "high", + "range" => "high", + "status" => "medium", + "cvss" => "medium", + "reference" => "low", + _ => "info", + }; + + private static string DeterminePrimitiveKinds(RangePrimitives? primitives) + { + return primitives is null ? "none" : primitives.GetCoverageTag(); + } + + private static string NormalizeMask(IReadOnlyList? fieldMask) + { + if (fieldMask is not { Count: > 0 }) + { + return string.Empty; + } + + if (fieldMask.Count == 1) + { + return fieldMask[0]; + } + + var ordered = fieldMask + .Where(static value => !string.IsNullOrWhiteSpace(value)) + .Select(static value => value.Trim().ToLowerInvariant()) + .Distinct(StringComparer.Ordinal) + .OrderBy(static value => value, StringComparer.Ordinal) + .ToArray(); + + return string.Join('|', ordered); + } + +} diff --git a/src/StellaOps.Feedser.Models/RangePrimitives.cs b/src/StellaOps.Feedser.Models/RangePrimitives.cs new file mode 100644 index 00000000..e77c208f --- /dev/null +++ b/src/StellaOps.Feedser.Models/RangePrimitives.cs @@ -0,0 +1,161 @@ +using System; +using System.Collections.Generic; + +namespace StellaOps.Feedser.Models; + +/// +/// Optional structured representations of range semantics attached to . +/// +public sealed record RangePrimitives( + SemVerPrimitive? SemVer, + NevraPrimitive? Nevra, + EvrPrimitive? Evr, + IReadOnlyDictionary? VendorExtensions) +{ + public bool HasVendorExtensions => VendorExtensions is { Count: > 0 }; + + public string GetCoverageTag() + { + var kinds = new List(3); + if (SemVer is not null) + { + kinds.Add("semver"); + } + + if (Nevra is not null) + { + kinds.Add("nevra"); + } + + if (Evr is not null) + { + kinds.Add("evr"); + } + + if (kinds.Count == 0) + { + return HasVendorExtensions ? "vendor" : "none"; + } + + kinds.Sort(StringComparer.Ordinal); + return string.Join('+', kinds); + } +} + +/// +/// Structured SemVer metadata for a version range. +/// +public sealed record SemVerPrimitive( + string? Introduced, + bool IntroducedInclusive, + string? Fixed, + bool FixedInclusive, + string? LastAffected, + bool LastAffectedInclusive, + string? ConstraintExpression, + string? ExactValue = null) +{ + public string Style => DetermineStyle( + Introduced, + IntroducedInclusive, + Fixed, + FixedInclusive, + LastAffected, + LastAffectedInclusive, + ConstraintExpression, + ExactValue); + + private static string DetermineStyle( + string? introduced, + bool introducedInclusive, + string? fixedVersion, + bool fixedInclusive, + string? lastAffected, + bool lastAffectedInclusive, + string? constraintExpression, + string? exactValue) + { + if (!string.IsNullOrWhiteSpace(exactValue)) + { + return SemVerPrimitiveStyles.Exact; + } + + var hasIntroduced = !string.IsNullOrWhiteSpace(introduced); + var hasFixed = !string.IsNullOrWhiteSpace(fixedVersion); + var hasLast = !string.IsNullOrWhiteSpace(lastAffected); + + if (hasIntroduced && hasFixed) + { + return SemVerPrimitiveStyles.Range; + } + + if (hasIntroduced) + { + return introducedInclusive + ? SemVerPrimitiveStyles.GreaterThanOrEqual + : SemVerPrimitiveStyles.GreaterThan; + } + + if (hasFixed) + { + return fixedInclusive + ? SemVerPrimitiveStyles.LessThanOrEqual + : SemVerPrimitiveStyles.LessThan; + } + + if (hasLast) + { + return lastAffectedInclusive + ? SemVerPrimitiveStyles.LessThanOrEqual + : SemVerPrimitiveStyles.LessThan; + } + + return string.IsNullOrWhiteSpace(constraintExpression) + ? SemVerPrimitiveStyles.Range + : SemVerPrimitiveStyles.Range; + } +} + +public static class SemVerPrimitiveStyles +{ + public const string Range = "range"; + public const string Exact = "exact"; + public const string LessThan = "lessThan"; + public const string LessThanOrEqual = "lessThanOrEqual"; + public const string GreaterThan = "greaterThan"; + public const string GreaterThanOrEqual = "greaterThanOrEqual"; +} + +/// +/// Structured NEVRA metadata for a version range. +/// +public sealed record NevraPrimitive( + NevraComponent? Introduced, + NevraComponent? Fixed, + NevraComponent? LastAffected); + +/// +/// Structured Debian EVR metadata for a version range. +/// +public sealed record EvrPrimitive( + EvrComponent? Introduced, + EvrComponent? Fixed, + EvrComponent? LastAffected); + +/// +/// Normalized NEVRA component. +/// +public sealed record NevraComponent( + string Name, + int Epoch, + string Version, + string Release, + string? Architecture); + +/// +/// Normalized EVR component (epoch:upstream revision). +/// +public sealed record EvrComponent( + int Epoch, + string UpstreamVersion, + string? Revision); diff --git a/src/StellaOps.Feedser.Models/SemVerPrimitiveExtensions.cs b/src/StellaOps.Feedser.Models/SemVerPrimitiveExtensions.cs new file mode 100644 index 00000000..3e96ce00 --- /dev/null +++ b/src/StellaOps.Feedser.Models/SemVerPrimitiveExtensions.cs @@ -0,0 +1,88 @@ +using System; + +namespace StellaOps.Feedser.Models; + +/// +/// Helper extensions for converting values into normalized rules. +/// +public static class SemVerPrimitiveExtensions +{ + public static NormalizedVersionRule? ToNormalizedVersionRule(this SemVerPrimitive? primitive, string? notes = null) + { + if (primitive is null) + { + return null; + } + + var trimmedNotes = Validation.TrimToNull(notes); + var scheme = NormalizedVersionSchemes.SemVer; + + if (!string.IsNullOrWhiteSpace(primitive.ExactValue)) + { + return new NormalizedVersionRule( + scheme, + NormalizedVersionRuleTypes.Exact, + value: primitive.ExactValue, + notes: trimmedNotes); + } + + var introduced = Validation.TrimToNull(primitive.Introduced); + var fixedVersion = Validation.TrimToNull(primitive.Fixed); + var lastAffected = Validation.TrimToNull(primitive.LastAffected); + + if (!string.IsNullOrEmpty(introduced) && !string.IsNullOrEmpty(fixedVersion)) + { + return new NormalizedVersionRule( + scheme, + NormalizedVersionRuleTypes.Range, + min: introduced, + minInclusive: primitive.IntroducedInclusive, + max: fixedVersion, + maxInclusive: primitive.FixedInclusive, + notes: trimmedNotes ?? primitive.ConstraintExpression); + } + + if (!string.IsNullOrEmpty(introduced) && string.IsNullOrEmpty(fixedVersion) && string.IsNullOrEmpty(lastAffected)) + { + var type = primitive.IntroducedInclusive ? NormalizedVersionRuleTypes.GreaterThanOrEqual : NormalizedVersionRuleTypes.GreaterThan; + return new NormalizedVersionRule( + scheme, + type, + min: introduced, + minInclusive: primitive.IntroducedInclusive, + notes: trimmedNotes); + } + + if (!string.IsNullOrEmpty(fixedVersion)) + { + var type = primitive.FixedInclusive ? NormalizedVersionRuleTypes.LessThanOrEqual : NormalizedVersionRuleTypes.LessThan; + return new NormalizedVersionRule( + scheme, + type, + max: fixedVersion, + maxInclusive: primitive.FixedInclusive, + notes: trimmedNotes); + } + + if (!string.IsNullOrEmpty(lastAffected)) + { + var type = primitive.LastAffectedInclusive ? NormalizedVersionRuleTypes.LessThanOrEqual : NormalizedVersionRuleTypes.LessThan; + return new NormalizedVersionRule( + scheme, + type, + max: lastAffected, + maxInclusive: primitive.LastAffectedInclusive, + notes: trimmedNotes); + } + + if (!string.IsNullOrWhiteSpace(primitive.ConstraintExpression)) + { + return new NormalizedVersionRule( + scheme, + NormalizedVersionRuleTypes.Range, + notes: trimmedNotes ?? primitive.ConstraintExpression); + } + + return null; + } +} diff --git a/src/StellaOps.Feedser.Models/SeverityNormalization.cs b/src/StellaOps.Feedser.Models/SeverityNormalization.cs new file mode 100644 index 00000000..280d528c --- /dev/null +++ b/src/StellaOps.Feedser.Models/SeverityNormalization.cs @@ -0,0 +1,152 @@ +using System; +using System.Collections.Generic; +using System.Text; + +namespace StellaOps.Feedser.Models; + +/// +/// Provides helpers to normalize vendor-provided severity labels into canonical values. +/// +public static class SeverityNormalization +{ + private static readonly IReadOnlyDictionary SeverityMap = new Dictionary + { + ["critical"] = "critical", + ["crit"] = "critical", + ["sevcritical"] = "critical", + ["extreme"] = "critical", + ["verycritical"] = "critical", + ["veryhigh"] = "critical", + ["p0"] = "critical", + ["priority0"] = "critical", + ["high"] = "high", + ["sevhigh"] = "high", + ["important"] = "high", + ["severe"] = "high", + ["major"] = "high", + ["urgent"] = "high", + ["elevated"] = "high", + ["p1"] = "high", + ["priority1"] = "high", + ["medium"] = "medium", + ["moderate"] = "medium", + ["normal"] = "medium", + ["avg"] = "medium", + ["average"] = "medium", + ["standard"] = "medium", + ["p2"] = "medium", + ["priority2"] = "medium", + ["low"] = "low", + ["minor"] = "low", + ["minimal"] = "low", + ["limited"] = "low", + ["p3"] = "low", + ["priority3"] = "low", + ["informational"] = "informational", + ["info"] = "informational", + ["informative"] = "informational", + ["notice"] = "informational", + ["advisory"] = "informational", + ["none"] = "none", + ["negligible"] = "none", + ["insignificant"] = "none", + ["notapplicable"] = "none", + ["na"] = "none", + ["unknown"] = "unknown", + ["undetermined"] = "unknown", + ["notdefined"] = "unknown", + ["notspecified"] = "unknown", + ["pending"] = "unknown", + ["tbd"] = "unknown", + }; + + private static readonly char[] TokenSeparators = + { + ' ', + '/', + '\\', + '-', + '_', + ',', + ';', + ':', + '(', + ')', + '[', + ']', + '{', + '}', + '|', + '+', + '&' + }; + + public static readonly IReadOnlyCollection CanonicalLevels = new[] + { + "critical", + "high", + "medium", + "low", + "informational", + "none", + "unknown", + }; + + public static string? Normalize(string? severity) + { + if (string.IsNullOrWhiteSpace(severity)) + { + return null; + } + + var trimmed = severity.Trim(); + + if (TryNormalizeToken(trimmed, out var mapped)) + { + return mapped; + } + + foreach (var token in trimmed.Split(TokenSeparators, StringSplitOptions.RemoveEmptyEntries | StringSplitOptions.TrimEntries)) + { + if (TryNormalizeToken(token, out mapped)) + { + return mapped; + } + } + + return trimmed.ToLowerInvariant(); + } + + private static bool TryNormalizeToken(string value, out string mapped) + { + var normalized = NormalizeToken(value); + if (normalized.Length == 0) + { + mapped = string.Empty; + return false; + } + + if (!SeverityMap.TryGetValue(normalized, out var mappedValue)) + { + mapped = string.Empty; + return false; + } + + mapped = mappedValue; + return true; + } + + private static string NormalizeToken(string value) + { + var builder = new StringBuilder(value.Length); + foreach (var ch in value) + { + if (char.IsLetterOrDigit(ch)) + { + builder.Append(char.ToLowerInvariant(ch)); + } + } + + return builder.ToString(); + } +} diff --git a/src/StellaOps.Feedser.Models/SnapshotSerializer.cs b/src/StellaOps.Feedser.Models/SnapshotSerializer.cs new file mode 100644 index 00000000..101c044f --- /dev/null +++ b/src/StellaOps.Feedser.Models/SnapshotSerializer.cs @@ -0,0 +1,27 @@ +using System.Text; +using System.Text.Json; + +namespace StellaOps.Feedser.Models; + +/// +/// Helper for tests/fixtures that need deterministic JSON snapshots. +/// +public static class SnapshotSerializer +{ + public static string ToSnapshot(T value) + => CanonicalJsonSerializer.SerializeIndented(value); + + public static void AppendSnapshot(StringBuilder builder, T value) + { + ArgumentNullException.ThrowIfNull(builder); + builder.AppendLine(ToSnapshot(value)); + } + + public static async Task WriteSnapshotAsync(Stream destination, T value, CancellationToken cancellationToken = default) + { + ArgumentNullException.ThrowIfNull(destination); + await using var writer = new StreamWriter(destination, new UTF8Encoding(encoderShouldEmitUTF8Identifier: false), leaveOpen: true); + await writer.WriteAsync(ToSnapshot(value).AsMemory(), cancellationToken).ConfigureAwait(false); + await writer.FlushAsync().ConfigureAwait(false); + } +} diff --git a/src/StellaOps.Feedser.Models/StellaOps.Feedser.Models.csproj b/src/StellaOps.Feedser.Models/StellaOps.Feedser.Models.csproj new file mode 100644 index 00000000..1d9208cf --- /dev/null +++ b/src/StellaOps.Feedser.Models/StellaOps.Feedser.Models.csproj @@ -0,0 +1,12 @@ + + + net10.0 + preview + enable + enable + true + + + + + diff --git a/src/StellaOps.Feedser.Models/TASKS.md b/src/StellaOps.Feedser.Models/TASKS.md new file mode 100644 index 00000000..c34ab18c --- /dev/null +++ b/src/StellaOps.Feedser.Models/TASKS.md @@ -0,0 +1,18 @@ +# TASKS +| Task | Owner(s) | Depends on | Notes | +|---|---|---|---| +|Canonical JSON serializer with stable ordering|BE-Merge|Models|DONE – `CanonicalJsonSerializer` ensures deterministic property ordering.| +|Equality/comparison helpers for ranges|BE-Merge|Models|DONE – added `AffectedVersionRangeComparer` & equality comparer.| +|Type enums/constants for AffectedPackage.Type|BE-Merge|Models|DONE – introduced `AffectedPackageTypes`.| +|Validation helpers (lightweight)|BE-Merge|Models|DONE – added `Validation` static helpers and URL guard.| +|Snapshot serializer for tests|QA|Models|DONE – `SnapshotSerializer` emits canonical JSON.| +|Docs: field provenance guidelines|BE-Merge|Models|DONE – see `PROVENANCE_GUIDELINES.md`.| +|Canonical record definitions kept in sync|BE-Merge|Models|DONE – documented in `CANONICAL_RECORDS.md`; update alongside model changes.| +|Alias scheme registry and validation helpers|BE-Merge|Models|DONE – see `AliasSchemes` & `AliasSchemeRegistry` plus validation integration/tests.| +|Range primitives for SemVer/EVR/NEVRA metadata|BE-Merge|Models|DOING – helpers (`RangePrimitives.GetCoverageTag`, diagnostics tags) landed; remaining connectors (Debian, SUSE, Ubuntu, Apple, Adobe, etc.) still need structured primitives/EVR population.| +|Provenance envelope field masks|BE-Merge|Models|DONE – `AdvisoryProvenance.fieldMask` added with diagnostics/tests/docs refreshed; connectors can now emit canonical masks for QA dashboards.| +|Backward-compatibility playbook|BE-Merge, QA|Models|DONE – see `BACKWARD_COMPATIBILITY.md` for evolution policy/test checklist.| +|Golden canonical examples|QA|Models|DONE – added `/p:UpdateGoldens=true` test hook wiring `UPDATE_GOLDENS=1` so canonical fixtures regenerate via `dotnet test`; docs/tests unchanged.| +|Serialization determinism regression tests|QA|Models|DONE – locale-stability tests hash canonical serializer output across multiple cultures and runs.| +|Severity normalization helpers|BE-Merge|Models|DONE – helper now normalizes compound vendor labels/priority tiers with expanded synonym coverage and regression tests.| +|AffectedPackage status glossary & guardrails|BE-Merge|Models|DONE – catalog now exposes deterministic listing, TryNormalize helpers, and synonym coverage for vendor phrases (not vulnerable, workaround available, etc.).| diff --git a/src/StellaOps.Feedser.Models/Validation.cs b/src/StellaOps.Feedser.Models/Validation.cs new file mode 100644 index 00000000..6f1b0bdd --- /dev/null +++ b/src/StellaOps.Feedser.Models/Validation.cs @@ -0,0 +1,57 @@ +using System.Diagnostics.CodeAnalysis; +using System.Text.RegularExpressions; + +namespace StellaOps.Feedser.Models; + +/// +/// Lightweight validation helpers shared across canonical model constructors. +/// +public static partial class Validation +{ + public static string EnsureNotNullOrWhiteSpace(string value, string paramName) + { + if (string.IsNullOrWhiteSpace(value)) + { + throw new ArgumentException($"Value cannot be null or whitespace.", paramName); + } + + return value.Trim(); + } + + public static string? TrimToNull(string? value) + => string.IsNullOrWhiteSpace(value) ? null : value.Trim(); + + public static bool LooksLikeHttpUrl(string? value) + => value is not null && Uri.TryCreate(value, UriKind.Absolute, out var uri) && (uri.Scheme is "http" or "https"); + + public static bool TryNormalizeAlias(string? value, [NotNullWhen(true)] out string? normalized) + { + normalized = TrimToNull(value); + if (normalized is null) + { + return false; + } + + if (AliasSchemeRegistry.TryNormalize(normalized, out var canonical, out _)) + { + normalized = canonical; + } + + return true; + } + + public static bool TryNormalizeIdentifier(string? value, [NotNullWhen(true)] out string? normalized) + { + normalized = TrimToNull(value); + return normalized is not null; + } + + [GeneratedRegex(@"\s+")] + private static partial Regex CollapseWhitespaceRegex(); + + public static string CollapseWhitespace(string value) + { + ArgumentNullException.ThrowIfNull(value); + return CollapseWhitespaceRegex().Replace(value, " ").Trim(); + } +} diff --git a/src/StellaOps.Feedser.Normalization.Tests/CpeNormalizerTests.cs b/src/StellaOps.Feedser.Normalization.Tests/CpeNormalizerTests.cs new file mode 100644 index 00000000..4fdf8a45 --- /dev/null +++ b/src/StellaOps.Feedser.Normalization.Tests/CpeNormalizerTests.cs @@ -0,0 +1,70 @@ +using StellaOps.Feedser.Normalization.Identifiers; + +namespace StellaOps.Feedser.Normalization.Tests; + +public sealed class CpeNormalizerTests +{ + [Fact] + public void TryNormalizeCpe_Preserves2Dot3Format() + { + var input = "cpe:2.3:A:Example:Product:1.0:*:*:*:*:*:*:*"; + + var success = IdentifierNormalizer.TryNormalizeCpe(input, out var normalized); + + Assert.True(success); + Assert.Equal("cpe:2.3:a:example:product:1.0:*:*:*:*:*:*:*", normalized); + } + + [Fact] + public void TryNormalizeCpe_UpgradesUriBinding() + { + var input = "cpe:/o:RedHat:Enterprise_Linux:8"; + + var success = IdentifierNormalizer.TryNormalizeCpe(input, out var normalized); + + Assert.True(success); + Assert.Equal("cpe:2.3:o:redhat:enterprise_linux:8:*:*:*:*:*:*:*", normalized); + } + + [Fact] + public void TryNormalizeCpe_InvalidInputReturnsFalse() + { + var success = IdentifierNormalizer.TryNormalizeCpe("not-a-cpe", out var normalized); + + Assert.False(success); + Assert.Null(normalized); + } + + [Fact] + public void TryNormalizeCpe_DecodesPercentEncodingAndEscapes() + { + var input = "cpe:/a:Example%20Corp:Widget%2fSuite:1.0:update:%7e:%2a"; + + var success = IdentifierNormalizer.TryNormalizeCpe(input, out var normalized); + + Assert.True(success); + Assert.Equal(@"cpe:2.3:a:example\ corp:widget\/suite:1.0:update:*:*:*:*:*:*", normalized); + } + + [Fact] + public void TryNormalizeCpe_ExpandsEditionFields() + { + var input = "cpe:/a:Vendor:Product:1.0:update:~pro~~windows~~:en-US"; + + var success = IdentifierNormalizer.TryNormalizeCpe(input, out var normalized); + + Assert.True(success); + Assert.Equal("cpe:2.3:a:vendor:product:1.0:update:*:en-us:pro:*:windows:*", normalized); + } + + [Fact] + public void TryNormalizeCpe_PreservesEscapedCharactersIn23() + { + var input = @"cpe:2.3:a:example:printer\/:1.2.3:*:*:*:*:*:*:*"; + + var success = IdentifierNormalizer.TryNormalizeCpe(input, out var normalized); + + Assert.True(success); + Assert.Equal(@"cpe:2.3:a:example:printer\/:1.2.3:*:*:*:*:*:*:*", normalized); + } +} diff --git a/src/StellaOps.Feedser.Normalization.Tests/CvssMetricNormalizerTests.cs b/src/StellaOps.Feedser.Normalization.Tests/CvssMetricNormalizerTests.cs new file mode 100644 index 00000000..1635e078 --- /dev/null +++ b/src/StellaOps.Feedser.Normalization.Tests/CvssMetricNormalizerTests.cs @@ -0,0 +1,52 @@ +using StellaOps.Feedser.Models; +using StellaOps.Feedser.Normalization.Cvss; + +namespace StellaOps.Feedser.Normalization.Tests; + +public sealed class CvssMetricNormalizerTests +{ + [Fact] + public void TryNormalize_ComputesCvss31Defaults() + { + var vector = "CVSS:3.1/AV:N/AC:L/PR:N/UI:N/S:U/C:H/I:H/A:H"; + + var success = CvssMetricNormalizer.TryNormalize(null, vector, null, null, out var normalized); + + Assert.True(success); + Assert.Equal("3.1", normalized.Version); + Assert.Equal(vector, normalized.Vector); + Assert.Equal(9.8, normalized.BaseScore); + Assert.Equal("critical", normalized.BaseSeverity); + + var provenance = new AdvisoryProvenance("nvd", "cvss", "https://example", DateTimeOffset.UnixEpoch); + var metric = normalized.ToModel(provenance); + Assert.Equal("3.1", metric.Version); + Assert.Equal(vector, metric.Vector); + Assert.Equal(9.8, metric.BaseScore); + Assert.Equal("critical", metric.BaseSeverity); + Assert.Equal(provenance, metric.Provenance); + } + + [Fact] + public void TryNormalize_NormalizesCvss20Severity() + { + var vector = "AV:N/AC:M/Au:S/C:P/I:P/A:P"; + + var success = CvssMetricNormalizer.TryNormalize("2.0", vector, 6.4, "MEDIUM", out var normalized); + + Assert.True(success); + Assert.Equal("2.0", normalized.Version); + Assert.Equal("CVSS:2.0/AV:N/AC:M/AU:S/C:P/I:P/A:P", normalized.Vector); + Assert.Equal(6.0, normalized.BaseScore); + Assert.Equal("medium", normalized.BaseSeverity); + } + + [Fact] + public void TryNormalize_ReturnsFalseWhenVectorMissing() + { + var success = CvssMetricNormalizer.TryNormalize("3.1", string.Empty, 9.8, "CRITICAL", out var normalized); + + Assert.False(success); + Assert.Equal(default, normalized); + } +} diff --git a/src/StellaOps.Feedser.Normalization.Tests/DebianEvrParserTests.cs b/src/StellaOps.Feedser.Normalization.Tests/DebianEvrParserTests.cs new file mode 100644 index 00000000..dbc4f4a9 --- /dev/null +++ b/src/StellaOps.Feedser.Normalization.Tests/DebianEvrParserTests.cs @@ -0,0 +1,31 @@ +using StellaOps.Feedser.Normalization.Distro; + +namespace StellaOps.Feedser.Normalization.Tests; + +public sealed class DebianEvrParserTests +{ + [Fact] + public void ToCanonicalString_RoundTripsExplicitEpoch() + { + var parsed = DebianEvr.Parse(" 1:1.2.3-1 "); + + Assert.Equal("1:1.2.3-1", parsed.Original); + Assert.Equal("1:1.2.3-1", parsed.ToCanonicalString()); + } + + [Fact] + public void ToCanonicalString_SuppressesZeroEpochWhenMissing() + { + var parsed = DebianEvr.Parse("1.2.3-1"); + + Assert.Equal("1.2.3-1", parsed.ToCanonicalString()); + } + + [Fact] + public void ToCanonicalString_HandlesMissingRevision() + { + var parsed = DebianEvr.Parse("2:4.5"); + + Assert.Equal("2:4.5", parsed.ToCanonicalString()); + } +} diff --git a/src/StellaOps.Feedser.Normalization.Tests/DescriptionNormalizerTests.cs b/src/StellaOps.Feedser.Normalization.Tests/DescriptionNormalizerTests.cs new file mode 100644 index 00000000..79b7f25b --- /dev/null +++ b/src/StellaOps.Feedser.Normalization.Tests/DescriptionNormalizerTests.cs @@ -0,0 +1,44 @@ +using StellaOps.Feedser.Normalization.Text; + +namespace StellaOps.Feedser.Normalization.Tests; + +public sealed class DescriptionNormalizerTests +{ + [Fact] + public void Normalize_RemovesMarkupAndCollapsesWhitespace() + { + var candidates = new[] + { + new LocalizedText("

    Hello\n\nworld!

    ", "en-US"), + }; + + var result = DescriptionNormalizer.Normalize(candidates); + + Assert.Equal("hello world!", result.Text.ToLowerInvariant()); + Assert.Equal("en", result.Language); + } + + [Fact] + public void Normalize_FallsBackToPreferredLanguage() + { + var candidates = new[] + { + new LocalizedText("Bonjour", "fr"), + new LocalizedText("Hello", "en-GB"), + }; + + var result = DescriptionNormalizer.Normalize(candidates); + + Assert.Equal("Hello", result.Text); + Assert.Equal("en", result.Language); + } + + [Fact] + public void Normalize_ReturnsDefaultWhenEmpty() + { + var result = DescriptionNormalizer.Normalize(Array.Empty()); + + Assert.Equal(string.Empty, result.Text); + Assert.Equal("en", result.Language); + } +} diff --git a/src/StellaOps.Feedser.Normalization.Tests/NevraParserTests.cs b/src/StellaOps.Feedser.Normalization.Tests/NevraParserTests.cs new file mode 100644 index 00000000..6a16797d --- /dev/null +++ b/src/StellaOps.Feedser.Normalization.Tests/NevraParserTests.cs @@ -0,0 +1,64 @@ +using StellaOps.Feedser.Normalization.Distro; + +namespace StellaOps.Feedser.Normalization.Tests; + +public sealed class NevraParserTests +{ + [Fact] + public void ToCanonicalString_RoundTripsTrimmedInput() + { + var parsed = Nevra.Parse(" kernel-0:4.18.0-80.el8.x86_64 "); + + Assert.Equal("kernel-0:4.18.0-80.el8.x86_64", parsed.Original); + Assert.Equal("kernel-0:4.18.0-80.el8.x86_64", parsed.ToCanonicalString()); + } + + [Fact] + public void ToCanonicalString_ReconstructsKnownArchitecture() + { + var parsed = Nevra.Parse("bash-5.2.15-3.el9_4.arm64"); + + Assert.Equal("bash-5.2.15-3.el9_4.arm64", parsed.ToCanonicalString()); + } + + [Fact] + public void ToCanonicalString_HandlesMissingArchitecture() + { + var parsed = Nevra.Parse("openssl-libs-1:1.1.1k-7.el8"); + + Assert.Equal("openssl-libs-1:1.1.1k-7.el8", parsed.ToCanonicalString()); + } + + [Fact] + public void TryParse_ReturnsTrueForExplicitZeroEpoch() + { + var success = Nevra.TryParse("glibc-0:2.36-8.el9.x86_64", out var nevra); + + Assert.True(success); + Assert.NotNull(nevra); + Assert.True(nevra!.HasExplicitEpoch); + Assert.Equal(0, nevra.Epoch); + Assert.Equal("glibc-0:2.36-8.el9.x86_64", nevra.ToCanonicalString()); + } + + [Fact] + public void TryParse_IgnoresUnknownArchitectureSuffix() + { + var success = Nevra.TryParse("package-1.0-1.el9.weirdarch", out var nevra); + + Assert.True(success); + Assert.NotNull(nevra); + Assert.Null(nevra!.Architecture); + Assert.Equal("package-1.0-1.el9.weirdarch", nevra.Original); + Assert.Equal("package-1.0-1.el9.weirdarch", nevra.ToCanonicalString()); + } + + [Fact] + public void TryParse_ReturnsFalseForMalformedNevra() + { + var success = Nevra.TryParse("bad-format", out var nevra); + + Assert.False(success); + Assert.Null(nevra); + } +} diff --git a/src/StellaOps.Feedser.Normalization.Tests/PackageUrlNormalizerTests.cs b/src/StellaOps.Feedser.Normalization.Tests/PackageUrlNormalizerTests.cs new file mode 100644 index 00000000..99599d30 --- /dev/null +++ b/src/StellaOps.Feedser.Normalization.Tests/PackageUrlNormalizerTests.cs @@ -0,0 +1,44 @@ +using System.Linq; +using StellaOps.Feedser.Normalization.Identifiers; + +namespace StellaOps.Feedser.Normalization.Tests; + +public sealed class PackageUrlNormalizerTests +{ + [Fact] + public void TryNormalizePackageUrl_LowersTypeAndNamespace() + { + var input = "pkg:NPM/Acme/Widget@1.0.0?Arch=X86_64"; + + var success = IdentifierNormalizer.TryNormalizePackageUrl(input, out var normalized, out var parsed); + + Assert.True(success); + Assert.Equal("pkg:npm/acme/widget@1.0.0?arch=X86_64", normalized); + Assert.NotNull(parsed); + Assert.Equal("npm", parsed!.Type); + Assert.Equal(new[] { "acme" }, parsed.NamespaceSegments.ToArray()); + Assert.Equal("widget", parsed.Name); + } + + [Fact] + public void TryNormalizePackageUrl_OrdersQualifiers() + { + var input = "pkg:deb/debian/openssl?distro=x%2Fy&arch=amd64"; + + var success = IdentifierNormalizer.TryNormalizePackageUrl(input, out var normalized, out _); + + Assert.True(success); + Assert.Equal("pkg:deb/debian/openssl?arch=amd64&distro=x%2Fy", normalized); + } + + [Fact] + public void TryNormalizePackageUrl_TrimsWhitespace() + { + var input = " pkg:pypi/Example/Package "; + + var success = IdentifierNormalizer.TryNormalizePackageUrl(input, out var normalized, out _); + + Assert.True(success); + Assert.Equal("pkg:pypi/example/package", normalized); + } +} diff --git a/src/StellaOps.Feedser.Normalization.Tests/SemVerRangeRuleBuilderTests.cs b/src/StellaOps.Feedser.Normalization.Tests/SemVerRangeRuleBuilderTests.cs new file mode 100644 index 00000000..4c79b97f --- /dev/null +++ b/src/StellaOps.Feedser.Normalization.Tests/SemVerRangeRuleBuilderTests.cs @@ -0,0 +1,113 @@ +using StellaOps.Feedser.Models; +using StellaOps.Feedser.Normalization.SemVer; +using Xunit; + +namespace StellaOps.Feedser.Normalization.Tests; + +public sealed class SemVerRangeRuleBuilderTests +{ + private const string Note = "spec:test"; + + [Theory] + [InlineData("< 1.5.0", null, NormalizedVersionRuleTypes.LessThan, null, true, "1.5.0", false, null, false)] + [InlineData(">= 1.0.0, < 2.0.0", null, NormalizedVersionRuleTypes.Range, "1.0.0", true, "2.0.0", false, null, false)] + [InlineData(">1.2.3, <=1.3.0", null, NormalizedVersionRuleTypes.Range, "1.2.3", false, null, false, "1.3.0", true)] + public void Build_ParsesCommonRanges( + string range, + string? patched, + string expectedNormalizedType, + string? expectedIntroduced, + bool expectedIntroducedInclusive, + string? expectedFixed, + bool expectedFixedInclusive, + string? expectedLastAffected, + bool expectedLastInclusive) + { + var results = SemVerRangeRuleBuilder.Build(range, patched, Note); + var result = Assert.Single(results); + + var primitive = result.Primitive; + Assert.Equal(expectedIntroduced, primitive.Introduced); + Assert.Equal(expectedIntroducedInclusive, primitive.IntroducedInclusive); + Assert.Equal(expectedFixed, primitive.Fixed); + Assert.Equal(expectedFixedInclusive, primitive.FixedInclusive); + Assert.Equal(expectedLastAffected, primitive.LastAffected); + Assert.Equal(expectedLastInclusive, primitive.LastAffectedInclusive); + + var normalized = result.NormalizedRule; + Assert.Equal(NormalizedVersionSchemes.SemVer, normalized.Scheme); + Assert.Equal(expectedNormalizedType, normalized.Type); + Assert.Equal(expectedIntroduced, normalized.Min); + Assert.Equal(expectedIntroduced is null ? (bool?)null : expectedIntroducedInclusive, normalized.MinInclusive); + Assert.Equal(expectedFixed ?? expectedLastAffected, normalized.Max); + Assert.Equal( + expectedFixed is not null ? expectedFixedInclusive : expectedLastInclusive, + normalized.MaxInclusive); + Assert.Equal(patched is null && expectedIntroduced is null && expectedFixed is null && expectedLastAffected is null ? null : Note, normalized.Notes); + } + + [Fact] + public void Build_UsesPatchedVersionWhenUpperBoundMissing() + { + var results = SemVerRangeRuleBuilder.Build(">= 4.0.0", "4.3.6", Note); + var result = Assert.Single(results); + + Assert.Equal("4.0.0", result.Primitive.Introduced); + Assert.Equal("4.3.6", result.Primitive.Fixed); + Assert.False(result.Primitive.FixedInclusive); + + var normalized = result.NormalizedRule; + Assert.Equal(NormalizedVersionRuleTypes.Range, normalized.Type); + Assert.Equal("4.0.0", normalized.Min); + Assert.True(normalized.MinInclusive); + Assert.Equal("4.3.6", normalized.Max); + Assert.False(normalized.MaxInclusive); + Assert.Equal(Note, normalized.Notes); + } + + [Theory] + [InlineData("^1.2.3", "1.2.3", "2.0.0")] + [InlineData("~1.2.3", "1.2.3", "1.3.0")] + [InlineData("~> 1.2", "1.2.0", "1.3.0")] + public void Build_HandlesCaretAndTilde(string range, string expectedMin, string expectedMax) + { + var results = SemVerRangeRuleBuilder.Build(range, null, Note); + var result = Assert.Single(results); + var normalized = result.NormalizedRule; + Assert.Equal(expectedMin, normalized.Min); + Assert.True(normalized.MinInclusive); + Assert.Equal(expectedMax, normalized.Max); + Assert.False(normalized.MaxInclusive); + Assert.Equal(NormalizedVersionRuleTypes.Range, normalized.Type); + } + + [Theory] + [InlineData("1.2.x", "1.2.0", "1.3.0")] + [InlineData("1.x", "1.0.0", "2.0.0")] + public void Build_HandlesWildcardNotation(string range, string expectedMin, string expectedMax) + { + var results = SemVerRangeRuleBuilder.Build(range, null, Note); + var result = Assert.Single(results); + Assert.Equal(expectedMin, result.Primitive.Introduced); + Assert.Equal(expectedMax, result.Primitive.Fixed); + + var normalized = result.NormalizedRule; + Assert.Equal(expectedMin, normalized.Min); + Assert.Equal(expectedMax, normalized.Max); + Assert.Equal(NormalizedVersionRuleTypes.Range, normalized.Type); + } + + [Fact] + public void Build_PreservesPreReleaseAndMetadataInExactRule() + { + var results = SemVerRangeRuleBuilder.Build("= 2.5.1-alpha.1+build.7", null, Note); + var result = Assert.Single(results); + + Assert.Equal("2.5.1-alpha.1+build.7", result.Primitive.ExactValue); + + var normalized = result.NormalizedRule; + Assert.Equal(NormalizedVersionRuleTypes.Exact, normalized.Type); + Assert.Equal("2.5.1-alpha.1+build.7", normalized.Value); + Assert.Equal(Note, normalized.Notes); + } +} diff --git a/src/StellaOps.Feedser.Normalization.Tests/StellaOps.Feedser.Normalization.Tests.csproj b/src/StellaOps.Feedser.Normalization.Tests/StellaOps.Feedser.Normalization.Tests.csproj new file mode 100644 index 00000000..0c41a48f --- /dev/null +++ b/src/StellaOps.Feedser.Normalization.Tests/StellaOps.Feedser.Normalization.Tests.csproj @@ -0,0 +1,11 @@ + + + net10.0 + enable + enable + + + + + + diff --git a/src/StellaOps.Feedser.Normalization/AssemblyInfo.cs b/src/StellaOps.Feedser.Normalization/AssemblyInfo.cs new file mode 100644 index 00000000..c367451d --- /dev/null +++ b/src/StellaOps.Feedser.Normalization/AssemblyInfo.cs @@ -0,0 +1,8 @@ +using System.Reflection; + +[assembly: AssemblyCompany("StellaOps")] +[assembly: AssemblyProduct("StellaOps.Feedser.Normalization")] +[assembly: AssemblyTitle("StellaOps.Feedser.Normalization")] +[assembly: AssemblyVersion("1.0.0.0")] +[assembly: AssemblyFileVersion("1.0.0.0")] +[assembly: AssemblyInformationalVersion("1.0.0")] diff --git a/src/StellaOps.Feedser.Normalization/Cvss/CvssMetricNormalizer.cs b/src/StellaOps.Feedser.Normalization/Cvss/CvssMetricNormalizer.cs new file mode 100644 index 00000000..b6aecb23 --- /dev/null +++ b/src/StellaOps.Feedser.Normalization/Cvss/CvssMetricNormalizer.cs @@ -0,0 +1,529 @@ +using System.Collections.Immutable; +using System.Linq; +using StellaOps.Feedser.Models; + +namespace StellaOps.Feedser.Normalization.Cvss; + +/// +/// Provides helpers to canonicalize CVSS vectors and fill in derived score/severity information. +/// +public static class CvssMetricNormalizer +{ + private static readonly string[] Cvss3BaseMetrics = { "AV", "AC", "PR", "UI", "S", "C", "I", "A" }; + private static readonly string[] Cvss2BaseMetrics = { "AV", "AC", "AU", "C", "I", "A" }; + + public static bool TryNormalize( + string? version, + string? vector, + double? baseScore, + string? baseSeverity, + out CvssNormalizedMetric metric) + { + metric = default; + if (string.IsNullOrWhiteSpace(vector)) + { + return false; + } + + var rawVector = vector.Trim(); + if (!TryDetermineVersion(version, rawVector, out var parsedVersion, out var vectorWithoutPrefix)) + { + return false; + } + + if (!TryParseMetrics(vectorWithoutPrefix, parsedVersion, out var canonicalVector, out var metrics)) + { + return false; + } + + if (!TryComputeBaseScore(parsedVersion, metrics, out var computedScore)) + { + return false; + } + + var normalizedScore = baseScore.HasValue + ? Math.Round(baseScore.Value, 1, MidpointRounding.AwayFromZero) + : computedScore; + + if (baseScore.HasValue && Math.Abs(normalizedScore - computedScore) > 0.2) + { + normalizedScore = computedScore; + } + + var severity = NormalizeSeverity(baseSeverity, parsedVersion) + ?? DetermineSeverity(normalizedScore, parsedVersion); + + metric = new CvssNormalizedMetric( + ToVersionString(parsedVersion), + canonicalVector, + normalizedScore, + severity); + + return true; + } + + private static bool TryDetermineVersion(string? versionToken, string vector, out CvssVersion version, out string withoutPrefix) + { + if (TryExtractVersionFromVector(vector, out version, out withoutPrefix)) + { + return true; + } + + if (!string.IsNullOrWhiteSpace(versionToken) && TryMapVersion(versionToken!, out version)) + { + withoutPrefix = StripPrefix(vector); + return true; + } + + var upper = vector.ToUpperInvariant(); + if (upper.Contains("PR:", StringComparison.Ordinal)) + { + version = CvssVersion.V31; + withoutPrefix = StripPrefix(vector); + return true; + } + + if (upper.Contains("AU:", StringComparison.Ordinal)) + { + version = CvssVersion.V20; + withoutPrefix = StripPrefix(vector); + return true; + } + + version = CvssVersion.V31; + withoutPrefix = StripPrefix(vector); + return true; + } + + private static string StripPrefix(string vector) + { + if (!vector.StartsWith("CVSS:", StringComparison.OrdinalIgnoreCase)) + { + return vector; + } + + var remainder = vector[5..]; + var slashIndex = remainder.IndexOf('/'); + return slashIndex >= 0 && slashIndex < remainder.Length - 1 + ? remainder[(slashIndex + 1)..] + : string.Empty; + } + + private static bool TryExtractVersionFromVector(string vector, out CvssVersion version, out string withoutPrefix) + { + withoutPrefix = vector; + if (!vector.StartsWith("CVSS:", StringComparison.OrdinalIgnoreCase)) + { + version = default; + return false; + } + + var remainder = vector[5..]; + var slashIndex = remainder.IndexOf('/'); + if (slashIndex <= 0 || slashIndex >= remainder.Length - 1) + { + version = CvssVersion.V31; + withoutPrefix = slashIndex > 0 && slashIndex < remainder.Length - 1 + ? remainder[(slashIndex + 1)..] + : string.Empty; + return false; + } + + var versionToken = remainder[..slashIndex]; + withoutPrefix = remainder[(slashIndex + 1)..]; + if (TryMapVersion(versionToken, out version)) + { + return true; + } + + version = CvssVersion.V31; + return false; + } + + private static bool TryMapVersion(string token, out CvssVersion version) + { + var trimmed = token.Trim(); + if (trimmed.Length == 0) + { + version = default; + return false; + } + + if (trimmed.StartsWith("v", StringComparison.OrdinalIgnoreCase)) + { + trimmed = trimmed[1..]; + } + + trimmed = trimmed switch + { + "3" or "3.1.0" or "3.1" => "3.1", + "3.0" or "3.0.0" => "3.0", + "2" or "2.0.0" => "2.0", + _ => trimmed, + }; + + version = trimmed switch + { + "2" or "2.0" => CvssVersion.V20, + "3.0" => CvssVersion.V30, + "3.1" => CvssVersion.V31, + _ => CvssVersion.Unknown, + }; + + return version != CvssVersion.Unknown; + } + + private static bool TryParseMetrics( + string vector, + CvssVersion version, + out string canonicalVector, + out ImmutableDictionary metrics) + { + canonicalVector = string.Empty; + var parsed = new Dictionary(StringComparer.OrdinalIgnoreCase); + var segments = vector.Split('/', StringSplitOptions.RemoveEmptyEntries); + if (segments.Length == 0) + { + metrics = ImmutableDictionary.Empty; + return false; + } + + foreach (var segment in segments) + { + var trimmed = segment.Trim(); + if (trimmed.Length == 0) + { + continue; + } + + var index = trimmed.IndexOf(':'); + if (index <= 0 || index == trimmed.Length - 1) + { + metrics = ImmutableDictionary.Empty; + return false; + } + + var key = trimmed[..index].Trim().ToUpperInvariant(); + var value = trimmed[(index + 1)..].Trim().ToUpperInvariant(); + if (key.Length == 0 || value.Length == 0) + { + metrics = ImmutableDictionary.Empty; + return false; + } + + parsed[key] = value; + } + + var required = version == CvssVersion.V20 ? Cvss2BaseMetrics : Cvss3BaseMetrics; + foreach (var metric in required) + { + if (!parsed.ContainsKey(metric)) + { + metrics = ImmutableDictionary.Empty; + return false; + } + } + + var canonicalSegments = new List(parsed.Count + 1); + foreach (var metric in required) + { + canonicalSegments.Add($"{metric}:{parsed[metric]}"); + } + + foreach (var entry in parsed.OrderBy(static pair => pair.Key, StringComparer.Ordinal)) + { + if (required.Contains(entry.Key)) + { + continue; + } + + canonicalSegments.Add($"{entry.Key}:{entry.Value}"); + } + + canonicalVector = $"CVSS:{ToVersionString(version)}/{string.Join('/', canonicalSegments)}"; + metrics = parsed.ToImmutableDictionary(StringComparer.OrdinalIgnoreCase); + return true; + } + + private static bool TryComputeBaseScore(CvssVersion version, IReadOnlyDictionary metrics, out double score) + { + return version switch + { + CvssVersion.V20 => TryComputeCvss2(metrics, out score), + CvssVersion.V30 or CvssVersion.V31 => TryComputeCvss3(metrics, out score), + _ => (score = 0) == 0, + }; + } + + private static bool TryComputeCvss3(IReadOnlyDictionary metrics, out double score) + { + try + { + var av = metrics["AV"] switch + { + "N" => 0.85, + "A" => 0.62, + "L" => 0.55, + "P" => 0.2, + _ => throw new InvalidOperationException(), + }; + + var ac = metrics["AC"] switch + { + "L" => 0.77, + "H" => 0.44, + _ => throw new InvalidOperationException(), + }; + + var scopeChanged = metrics["S"] switch + { + "U" => false, + "C" => true, + _ => throw new InvalidOperationException(), + }; + + var pr = metrics["PR"] switch + { + "N" => 0.85, + "L" => scopeChanged ? 0.68 : 0.62, + "H" => scopeChanged ? 0.5 : 0.27, + _ => throw new InvalidOperationException(), + }; + + var ui = metrics["UI"] switch + { + "N" => 0.85, + "R" => 0.62, + _ => throw new InvalidOperationException(), + }; + + var confidentiality = metrics["C"] switch + { + "N" => 0.0, + "L" => 0.22, + "H" => 0.56, + _ => throw new InvalidOperationException(), + }; + + var integrity = metrics["I"] switch + { + "N" => 0.0, + "L" => 0.22, + "H" => 0.56, + _ => throw new InvalidOperationException(), + }; + + var availability = metrics["A"] switch + { + "N" => 0.0, + "L" => 0.22, + "H" => 0.56, + _ => throw new InvalidOperationException(), + }; + + var impactSub = 1 - (1 - confidentiality) * (1 - integrity) * (1 - availability); + impactSub = Math.Clamp(impactSub, 0, 1); + + var impact = scopeChanged + ? 7.52 * (impactSub - 0.029) - 3.25 * Math.Pow(impactSub - 0.02, 15) + : 6.42 * impactSub; + + var exploitability = 8.22 * av * ac * pr * ui; + + if (impact <= 0) + { + score = 0; + return true; + } + + var baseScore = scopeChanged + ? Math.Min(1.08 * (impact + exploitability), 10) + : Math.Min(impact + exploitability, 10); + + score = RoundUp(baseScore); + return true; + } + catch (KeyNotFoundException) + { + score = 0; + return false; + } + catch (InvalidOperationException) + { + score = 0; + return false; + } + } + + private static bool TryComputeCvss2(IReadOnlyDictionary metrics, out double score) + { + try + { + var av = metrics["AV"] switch + { + "L" => 0.395, + "A" => 0.646, + "N" => 1.0, + _ => throw new InvalidOperationException(), + }; + + var ac = metrics["AC"] switch + { + "H" => 0.35, + "M" => 0.61, + "L" => 0.71, + _ => throw new InvalidOperationException(), + }; + + var authValue = metrics.TryGetValue("AU", out var primaryAuth) + ? primaryAuth + : metrics.TryGetValue("AUTH", out var fallbackAuth) + ? fallbackAuth + : null; + + if (string.IsNullOrEmpty(authValue)) + { + throw new InvalidOperationException(); + } + + var authentication = authValue switch + { + "M" => 0.45, + "S" => 0.56, + "N" => 0.704, + _ => throw new InvalidOperationException(), + }; + + var confidentiality = metrics["C"] switch + { + "N" => 0.0, + "P" => 0.275, + "C" => 0.660, + _ => throw new InvalidOperationException(), + }; + + var integrity = metrics["I"] switch + { + "N" => 0.0, + "P" => 0.275, + "C" => 0.660, + _ => throw new InvalidOperationException(), + }; + + var availability = metrics["A"] switch + { + "N" => 0.0, + "P" => 0.275, + "C" => 0.660, + _ => throw new InvalidOperationException(), + }; + + var impact = 10.41 * (1 - (1 - confidentiality) * (1 - integrity) * (1 - availability)); + var exploitability = 20 * av * ac * authentication; + var fImpact = impact == 0 ? 0.0 : 1.176; + var baseScore = ((0.6 * impact) + (0.4 * exploitability) - 1.5) * fImpact; + score = Math.Round(Math.Max(baseScore, 0), 1, MidpointRounding.AwayFromZero); + return true; + } + catch (KeyNotFoundException) + { + score = 0; + return false; + } + catch (InvalidOperationException) + { + score = 0; + return false; + } + } + + private static string DetermineSeverity(double score, CvssVersion version) + { + if (score <= 0) + { + return "none"; + } + + if (version == CvssVersion.V20) + { + if (score < 4.0) + { + return "low"; + } + + if (score < 7.0) + { + return "medium"; + } + + return "high"; + } + + if (score < 4.0) + { + return "low"; + } + + if (score < 7.0) + { + return "medium"; + } + + if (score < 9.0) + { + return "high"; + } + + return "critical"; + } + + private static string? NormalizeSeverity(string? severity, CvssVersion version) + { + if (string.IsNullOrWhiteSpace(severity)) + { + return null; + } + + var normalized = severity.Trim().ToLowerInvariant(); + return normalized switch + { + "none" or "informational" or "info" => "none", + "critical" when version != CvssVersion.V20 => "critical", + "critical" when version == CvssVersion.V20 => "high", + "high" => "high", + "medium" or "moderate" => "medium", + "low" => "low", + _ => null, + }; + } + + private static double RoundUp(double value) + { + return Math.Ceiling(value * 10.0) / 10.0; + } + + private static string ToVersionString(CvssVersion version) + => version switch + { + CvssVersion.V20 => "2.0", + CvssVersion.V30 => "3.0", + _ => "3.1", + }; + + private enum CvssVersion + { + Unknown = 0, + V20, + V30, + V31, + } +} + +/// +/// Represents a normalized CVSS metric ready for canonical serialization. +/// +public readonly record struct CvssNormalizedMetric(string Version, string Vector, double BaseScore, string BaseSeverity) +{ + public CvssMetric ToModel(AdvisoryProvenance provenance) + => new(Version, Vector, BaseScore, BaseSeverity, provenance); +} diff --git a/src/StellaOps.Feedser.Normalization/Distro/DebianEvr.cs b/src/StellaOps.Feedser.Normalization/Distro/DebianEvr.cs new file mode 100644 index 00000000..f563d869 --- /dev/null +++ b/src/StellaOps.Feedser.Normalization/Distro/DebianEvr.cs @@ -0,0 +1,127 @@ +using System.Globalization; + +namespace StellaOps.Feedser.Normalization.Distro; + +/// +/// Represents a Debian epoch:version-revision tuple and exposes parsing/formatting helpers. +/// +public sealed class DebianEvr +{ + private DebianEvr(int epoch, bool hasExplicitEpoch, string version, string revision, string original) + { + Epoch = epoch; + HasExplicitEpoch = hasExplicitEpoch; + Version = version; + Revision = revision; + Original = original; + } + + /// + /// Epoch segment (defaults to 0 when omitted). + /// + public int Epoch { get; } + + /// + /// Indicates whether an epoch segment was present explicitly. + /// + public bool HasExplicitEpoch { get; } + + /// + /// Version portion (without revision). + /// + public string Version { get; } + + /// + /// Revision portion (after the last dash). Empty when omitted. + /// + public string Revision { get; } + + /// + /// Trimmed EVR string supplied to . + /// + public string Original { get; } + + /// + /// Attempts to parse the provided value into a instance. + /// + public static bool TryParse(string? value, out DebianEvr? result) + { + result = null; + if (string.IsNullOrWhiteSpace(value)) + { + return false; + } + + var trimmed = value.Trim(); + var epoch = 0; + var hasExplicitEpoch = false; + var remainder = trimmed; + + var colonIndex = remainder.IndexOf(':'); + if (colonIndex >= 0) + { + if (colonIndex == 0) + { + return false; + } + + var epochPart = remainder[..colonIndex]; + if (!int.TryParse(epochPart, NumberStyles.Integer, CultureInfo.InvariantCulture, out epoch)) + { + return false; + } + + hasExplicitEpoch = true; + remainder = colonIndex < remainder.Length - 1 ? remainder[(colonIndex + 1)..] : string.Empty; + } + + if (string.IsNullOrEmpty(remainder)) + { + return false; + } + + var version = remainder; + var revision = string.Empty; + + var dashIndex = remainder.LastIndexOf('-'); + if (dashIndex > 0) + { + version = remainder[..dashIndex]; + revision = dashIndex < remainder.Length - 1 ? remainder[(dashIndex + 1)..] : string.Empty; + } + + if (string.IsNullOrEmpty(version)) + { + return false; + } + + result = new DebianEvr(epoch, hasExplicitEpoch, version, revision, trimmed); + return true; + } + + /// + /// Parses the provided value into a or throws . + /// + public static DebianEvr Parse(string value) + { + if (!TryParse(value, out var evr)) + { + throw new FormatException($"Input '{value}' is not a valid Debian EVR string."); + } + + return evr!; + } + + /// + /// Returns a canonical EVR string with trimmed components and normalized epoch/revision placement. + /// + public string ToCanonicalString() + { + var epochSegment = HasExplicitEpoch || Epoch > 0 ? $"{Epoch}:" : string.Empty; + var revisionSegment = string.IsNullOrEmpty(Revision) ? string.Empty : $"-{Revision}"; + return $"{epochSegment}{Version}{revisionSegment}"; + } + + /// + public override string ToString() => Original; +} diff --git a/src/StellaOps.Feedser.Normalization/Distro/Nevra.cs b/src/StellaOps.Feedser.Normalization/Distro/Nevra.cs new file mode 100644 index 00000000..daf61944 --- /dev/null +++ b/src/StellaOps.Feedser.Normalization/Distro/Nevra.cs @@ -0,0 +1,192 @@ +using System.Globalization; + +namespace StellaOps.Feedser.Normalization.Distro; + +/// +/// Represents a parsed NEVRA (Name-Epoch:Version-Release.Architecture) identifier and exposes helpers for canonical formatting. +/// +public sealed class Nevra +{ + private Nevra(string name, int epoch, bool hasExplicitEpoch, string version, string release, string? architecture, string original) + { + Name = name; + Epoch = epoch; + HasExplicitEpoch = hasExplicitEpoch; + Version = version; + Release = release; + Architecture = architecture; + Original = original; + } + + /// + /// Package name segment. + /// + public string Name { get; } + + /// + /// Epoch extracted from the NEVRA string (defaults to 0 when omitted). + /// + public int Epoch { get; } + + /// + /// Indicates whether an epoch segment was present explicitly (e.g. 0:). + /// + public bool HasExplicitEpoch { get; } + + /// + /// Version component (without epoch or release). + /// + public string Version { get; } + + /// + /// Release component (without architecture suffix). + /// + public string Release { get; } + + /// + /// Optional architecture suffix (e.g. x86_64, noarch). + /// + public string? Architecture { get; } + + /// + /// Trimmed NEVRA string supplied to . + /// + public string Original { get; } + + private static readonly ISet KnownArchitectures = new HashSet(StringComparer.OrdinalIgnoreCase) + { + "noarch", + "src", + "nosrc", + "x86_64", + "aarch64", + "armv7hl", + "armhfp", + "ppc64", + "ppc64le", + "ppc", + "s390", + "s390x", + "i386", + "i486", + "i586", + "i686", + "amd64", + "arm64", + "armv7l", + "armv6l", + "armv8l", + "armel", + "armhf", + "ia32e", + "loongarch64", + "mips", + "mips64", + "mips64le", + "mipsel", + "ppc32", + "ppc64p7", + "riscv64", + "sparc", + "sparc64" + }; + + /// + /// Attempts to parse the provided value into a instance. + /// + public static bool TryParse(string? value, out Nevra? result) + { + result = null; + if (string.IsNullOrWhiteSpace(value)) + { + return false; + } + + var trimmed = value.Trim(); + var releaseSeparator = trimmed.LastIndexOf('-'); + if (releaseSeparator <= 0 || releaseSeparator >= trimmed.Length - 1) + { + return false; + } + + var releasePart = trimmed[(releaseSeparator + 1)..]; + var nameVersionPart = trimmed[..releaseSeparator]; + + var versionSeparator = nameVersionPart.LastIndexOf('-'); + if (versionSeparator <= 0 || versionSeparator >= nameVersionPart.Length) + { + return false; + } + + var versionPart = nameVersionPart[(versionSeparator + 1)..]; + var namePart = nameVersionPart[..versionSeparator]; + + if (string.IsNullOrWhiteSpace(namePart)) + { + return false; + } + + string? architecture = null; + var release = releasePart; + var architectureSeparator = releasePart.LastIndexOf('.'); + if (architectureSeparator > 0 && architectureSeparator < releasePart.Length - 1) + { + var possibleArch = releasePart[(architectureSeparator + 1)..]; + if (KnownArchitectures.Contains(possibleArch)) + { + architecture = possibleArch; + release = releasePart[..architectureSeparator]; + } + } + + var version = versionPart; + var epoch = 0; + var hasExplicitEpoch = false; + var epochSeparator = versionPart.IndexOf(':'); + if (epochSeparator >= 0) + { + hasExplicitEpoch = true; + var epochPart = versionPart[..epochSeparator]; + version = epochSeparator < versionPart.Length - 1 ? versionPart[(epochSeparator + 1)..] : string.Empty; + + if (epochPart.Length > 0 && !int.TryParse(epochPart, NumberStyles.Integer, CultureInfo.InvariantCulture, out epoch)) + { + return false; + } + } + + if (string.IsNullOrWhiteSpace(version)) + { + return false; + } + + result = new Nevra(namePart, epoch, hasExplicitEpoch, version, release, architecture, trimmed); + return true; + } + + /// + /// Parses the provided value into a or throws . + /// + public static Nevra Parse(string value) + { + if (!TryParse(value, out var nevra)) + { + throw new FormatException($"Input '{value}' is not a valid NEVRA string."); + } + + return nevra!; + } + + /// + /// Returns a canonical NEVRA string with trimmed components and normalized epoch/architecture placement. + /// + public string ToCanonicalString() + { + var epochSegment = HasExplicitEpoch || Epoch > 0 ? $"{Epoch}:" : string.Empty; + var archSegment = string.IsNullOrWhiteSpace(Architecture) ? string.Empty : $".{Architecture}"; + return $"{Name}-{epochSegment}{Version}-{Release}{archSegment}"; + } + + /// + public override string ToString() => Original; +} diff --git a/src/StellaOps.Feedser.Normalization/Identifiers/Cpe23.cs b/src/StellaOps.Feedser.Normalization/Identifiers/Cpe23.cs new file mode 100644 index 00000000..bd92d20d --- /dev/null +++ b/src/StellaOps.Feedser.Normalization/Identifiers/Cpe23.cs @@ -0,0 +1,352 @@ +using System.Collections.Generic; +using System.Globalization; +using System.Text; + +namespace StellaOps.Feedser.Normalization.Identifiers; + +/// +/// Implements canonical normalization for CPE 2.3 identifiers (and URI binding conversion). +/// +internal static class Cpe23 +{ + private static readonly HashSet CharactersRequiringEscape = new(new[] + { + '\\', ':', '/', '?', '#', '[', ']', '@', '!', '$', '&', '"', '\'', '(', ')', '+', ',', ';', '=', '%', '*', + '<', '>', '|', '^', '`', '{', '}', '~' + }); + + public static bool TryNormalize(string? value, out string? normalized) + { + normalized = null; + if (string.IsNullOrWhiteSpace(value)) + { + return false; + } + + var trimmed = value.Trim(); + var components = SplitComponents(trimmed); + if (components.Count == 0) + { + return false; + } + + if (!components[0].Equals("cpe", StringComparison.OrdinalIgnoreCase)) + { + return false; + } + + if (components.Count >= 2 && components[1].Equals("2.3", StringComparison.OrdinalIgnoreCase)) + { + return TryNormalizeFrom23(components, out normalized); + } + + if (components.Count >= 2 && components[1].Length > 0 && components[1][0] == '/') + { + return TryNormalizeFrom22(components, out normalized); + } + + return false; + } + + private static bool TryNormalizeFrom23(IReadOnlyList components, out string? normalized) + { + normalized = null; + if (components.Count != 13) + { + return false; + } + + var part = NormalizePart(components[2]); + if (part is null) + { + return false; + } + + var normalizedComponents = new string[13]; + normalizedComponents[0] = "cpe"; + normalizedComponents[1] = "2.3"; + normalizedComponents[2] = part; + normalizedComponents[3] = NormalizeField(components[3], lower: true, decodeUri: false); + normalizedComponents[4] = NormalizeField(components[4], lower: true, decodeUri: false); + normalizedComponents[5] = NormalizeField(components[5], lower: false, decodeUri: false); + normalizedComponents[6] = NormalizeField(components[6], lower: false, decodeUri: false); + normalizedComponents[7] = NormalizeField(components[7], lower: false, decodeUri: false); + normalizedComponents[8] = NormalizeField(components[8], lower: false, decodeUri: false); + normalizedComponents[9] = NormalizeField(components[9], lower: false, decodeUri: false); + normalizedComponents[10] = NormalizeField(components[10], lower: false, decodeUri: false); + normalizedComponents[11] = NormalizeField(components[11], lower: false, decodeUri: false); + normalizedComponents[12] = NormalizeField(components[12], lower: false, decodeUri: false); + + normalized = string.Join(':', normalizedComponents); + return true; + } + + private static bool TryNormalizeFrom22(IReadOnlyList components, out string? normalized) + { + normalized = null; + if (components.Count < 2) + { + return false; + } + + var partComponent = components[1]; + if (partComponent.Length < 2 || partComponent[0] != '/') + { + return false; + } + + var part = NormalizePart(partComponent[1..]); + if (part is null) + { + return false; + } + + var vendor = NormalizeField(components.Count > 2 ? components[2] : null, lower: true, decodeUri: true); + var product = NormalizeField(components.Count > 3 ? components[3] : null, lower: true, decodeUri: true); + var version = NormalizeField(components.Count > 4 ? components[4] : null, lower: false, decodeUri: true); + var update = NormalizeField(components.Count > 5 ? components[5] : null, lower: false, decodeUri: true); + + var (edition, swEdition, targetSw, targetHw, other) = ExpandEdition(components.Count > 6 ? components[6] : null); + var language = NormalizeField(components.Count > 7 ? components[7] : null, lower: true, decodeUri: true); + + normalized = string.Join(':', new[] + { + "cpe", + "2.3", + part, + vendor, + product, + version, + update, + edition, + language, + swEdition, + targetSw, + targetHw, + other, + }); + + return true; + } + + private static string? NormalizePart(string value) + { + if (string.IsNullOrWhiteSpace(value)) + { + return null; + } + + var token = value.Trim().ToLowerInvariant(); + return token is "a" or "o" or "h" ? token : null; + } + + private static string NormalizeField(string? value, bool lower, bool decodeUri) + { + if (string.IsNullOrWhiteSpace(value)) + { + return "*"; + } + + var trimmed = value.Trim(); + if (trimmed is "*" or "-") + { + return trimmed; + } + + var decoded = decodeUri ? DecodeUriComponent(trimmed) : UnescapeComponent(trimmed); + if (decoded is "*" or "-") + { + return decoded; + } + + if (decoded.Length == 0) + { + return "*"; + } + + var normalized = lower ? decoded.ToLowerInvariant() : decoded; + return EscapeComponent(normalized); + } + + private static (string Edition, string SwEdition, string TargetSw, string TargetHw, string Other) ExpandEdition(string? value) + { + if (string.IsNullOrWhiteSpace(value)) + { + return ("*", "*", "*", "*", "*"); + } + + var trimmed = value.Trim(); + if (trimmed is "*" or "-") + { + return (trimmed, "*", "*", "*", "*"); + } + + var decoded = DecodeUriComponent(trimmed); + if (!decoded.StartsWith("~", StringComparison.Ordinal)) + { + return (NormalizeDecodedField(decoded, lower: false), "*", "*", "*", "*"); + } + + var segments = decoded.Split('~'); + var swEdition = segments.Length > 1 ? NormalizeDecodedField(segments[1], lower: false) : "*"; + var targetSw = segments.Length > 2 ? NormalizeDecodedField(segments[2], lower: false) : "*"; + var targetHw = segments.Length > 3 ? NormalizeDecodedField(segments[3], lower: false) : "*"; + var other = segments.Length > 4 ? NormalizeDecodedField(segments[4], lower: false) : "*"; + + return ("*", swEdition, targetSw, targetHw, other); + } + + private static string NormalizeDecodedField(string? value, bool lower) + { + if (string.IsNullOrWhiteSpace(value)) + { + return "*"; + } + + var trimmed = value.Trim(); + if (trimmed is "*" or "-") + { + return trimmed; + } + + var normalized = lower ? trimmed.ToLowerInvariant() : trimmed; + if (normalized is "*" or "-") + { + return normalized; + } + + return EscapeComponent(normalized); + } + + private static string UnescapeComponent(string value) + { + var builder = new StringBuilder(value.Length); + var escape = false; + foreach (var ch in value) + { + if (escape) + { + builder.Append(ch); + escape = false; + continue; + } + + if (ch == '\\') + { + escape = true; + continue; + } + + builder.Append(ch); + } + + if (escape) + { + builder.Append('\\'); + } + + return builder.ToString(); + } + + private static string EscapeComponent(string value) + { + if (value.Length == 0) + { + return value; + } + + var builder = new StringBuilder(value.Length * 2); + foreach (var ch in value) + { + if (RequiresEscape(ch)) + { + builder.Append('\\'); + } + + builder.Append(ch); + } + + return builder.ToString(); + } + + private static bool RequiresEscape(char ch) + { + if (char.IsLetterOrDigit(ch)) + { + return false; + } + + if (char.IsWhiteSpace(ch)) + { + return true; + } + + return ch switch + { + '_' or '-' or '.' => false, + // Keep wildcard markers literal only when entire component is wildcard handled earlier. + '*' => true, + _ => CharactersRequiringEscape.Contains(ch) + }; + } + + private static string DecodeUriComponent(string value) + { + var builder = new StringBuilder(value.Length); + for (var i = 0; i < value.Length; i++) + { + var ch = value[i]; + if (ch == '%' && i + 2 < value.Length && IsHex(value[i + 1]) && IsHex(value[i + 2])) + { + var hex = new string(new[] { value[i + 1], value[i + 2] }); + var decoded = (char)int.Parse(hex, NumberStyles.HexNumber, CultureInfo.InvariantCulture); + builder.Append(decoded); + i += 2; + } + else + { + builder.Append(ch); + } + } + + return builder.ToString(); + } + + private static bool IsHex(char ch) + => ch is >= '0' and <= '9' or >= 'A' and <= 'F' or >= 'a' and <= 'f'; + + private static List SplitComponents(string value) + { + var results = new List(); + var builder = new StringBuilder(); + var escape = false; + foreach (var ch in value) + { + if (escape) + { + builder.Append(ch); + escape = false; + continue; + } + + if (ch == '\\') + { + builder.Append(ch); + escape = true; + continue; + } + + if (ch == ':') + { + results.Add(builder.ToString()); + builder.Clear(); + continue; + } + + builder.Append(ch); + } + + results.Add(builder.ToString()); + return results; + } +} diff --git a/src/StellaOps.Feedser.Normalization/Identifiers/IdentifierNormalizer.cs b/src/StellaOps.Feedser.Normalization/Identifiers/IdentifierNormalizer.cs new file mode 100644 index 00000000..6b36081f --- /dev/null +++ b/src/StellaOps.Feedser.Normalization/Identifiers/IdentifierNormalizer.cs @@ -0,0 +1,32 @@ +namespace StellaOps.Feedser.Normalization.Identifiers; + +/// +/// Provides canonical normalization helpers for package identifiers. +/// +public static class IdentifierNormalizer +{ + public static bool TryNormalizePackageUrl(string? value, out string? normalized, out PackageUrl? packageUrl) + { + normalized = null; + packageUrl = null; + if (!PackageUrl.TryParse(value, out var parsed)) + { + return false; + } + + var canonical = parsed!.ToCanonicalString(); + normalized = canonical; + packageUrl = parsed; + return true; + } + + public static bool TryNormalizePackageUrl(string? value, out string? normalized) + { + return TryNormalizePackageUrl(value, out normalized, out _); + } + + public static bool TryNormalizeCpe(string? value, out string? normalized) + { + return Cpe23.TryNormalize(value, out normalized); + } +} diff --git a/src/StellaOps.Feedser.Normalization/Identifiers/PackageUrl.cs b/src/StellaOps.Feedser.Normalization/Identifiers/PackageUrl.cs new file mode 100644 index 00000000..fbfda254 --- /dev/null +++ b/src/StellaOps.Feedser.Normalization/Identifiers/PackageUrl.cs @@ -0,0 +1,299 @@ +using System.Collections.Immutable; +using System.Linq; +using System.Text; + +namespace StellaOps.Feedser.Normalization.Identifiers; + +/// +/// Represents a parsed Package URL (purl) identifier with canonical string rendering. +/// +public sealed class PackageUrl +{ + private PackageUrl( + string type, + ImmutableArray namespaceSegments, + string name, + string? version, + ImmutableArray> qualifiers, + ImmutableArray subpathSegments, + string original) + { + Type = type; + NamespaceSegments = namespaceSegments; + Name = name; + Version = version; + Qualifiers = qualifiers; + SubpathSegments = subpathSegments; + Original = original; + } + + public string Type { get; } + + public ImmutableArray NamespaceSegments { get; } + + public string Name { get; } + + public string? Version { get; } + + public ImmutableArray> Qualifiers { get; } + + public ImmutableArray SubpathSegments { get; } + + public string Original { get; } + + private static readonly HashSet LowerCaseNamespaceTypes = new(StringComparer.OrdinalIgnoreCase) + { + "maven", + "npm", + "pypi", + "nuget", + "composer", + "gem", + "apk", + "deb", + "rpm", + "oci", + }; + + private static readonly HashSet LowerCaseNameTypes = new(StringComparer.OrdinalIgnoreCase) + { + "npm", + "pypi", + "nuget", + "composer", + "gem", + "apk", + "deb", + "rpm", + "oci", + }; + + public static bool TryParse(string? value, out PackageUrl? packageUrl) + { + packageUrl = null; + if (string.IsNullOrWhiteSpace(value)) + { + return false; + } + + var trimmed = value.Trim(); + if (!trimmed.StartsWith("pkg:", StringComparison.OrdinalIgnoreCase)) + { + return false; + } + + var remainder = trimmed[4..]; + var firstSlash = remainder.IndexOf('/'); + if (firstSlash <= 0) + { + return false; + } + + var type = remainder[..firstSlash].Trim().ToLowerInvariant(); + remainder = remainder[(firstSlash + 1)..]; + + var subpathPart = string.Empty; + var subpathIndex = remainder.IndexOf('#'); + if (subpathIndex >= 0) + { + subpathPart = remainder[(subpathIndex + 1)..]; + remainder = remainder[..subpathIndex]; + } + + var qualifierPart = string.Empty; + var qualifierIndex = remainder.IndexOf('?'); + if (qualifierIndex >= 0) + { + qualifierPart = remainder[(qualifierIndex + 1)..]; + remainder = remainder[..qualifierIndex]; + } + + string? version = null; + var versionIndex = remainder.LastIndexOf('@'); + if (versionIndex >= 0) + { + version = remainder[(versionIndex + 1)..]; + remainder = remainder[..versionIndex]; + } + + if (string.IsNullOrWhiteSpace(remainder)) + { + return false; + } + + var rawSegments = remainder.Split('/', StringSplitOptions.RemoveEmptyEntries); + if (rawSegments.Length == 0) + { + return false; + } + + var shouldLowerNamespace = LowerCaseNamespaceTypes.Contains(type); + var shouldLowerName = LowerCaseNameTypes.Contains(type); + + var namespaceBuilder = ImmutableArray.CreateBuilder(Math.Max(0, rawSegments.Length - 1)); + for (var i = 0; i < rawSegments.Length - 1; i++) + { + var segment = Uri.UnescapeDataString(rawSegments[i].Trim()); + if (segment.Length == 0) + { + continue; + } + + if (shouldLowerNamespace) + { + segment = segment.ToLowerInvariant(); + } + + namespaceBuilder.Add(EscapePathSegment(segment)); + } + + var nameSegment = Uri.UnescapeDataString(rawSegments[^1].Trim()); + if (nameSegment.Length == 0) + { + return false; + } + + if (shouldLowerName) + { + nameSegment = nameSegment.ToLowerInvariant(); + } + + var canonicalName = EscapePathSegment(nameSegment); + var canonicalVersion = NormalizeComponent(version, escape: true, lowerCase: false); + var qualifiers = ParseQualifiers(qualifierPart); + var subpath = ParseSubpath(subpathPart); + + packageUrl = new PackageUrl( + type, + namespaceBuilder.ToImmutable(), + canonicalName, + canonicalVersion, + qualifiers, + subpath, + trimmed); + return true; + } + + public static PackageUrl Parse(string value) + { + if (!TryParse(value, out var parsed)) + { + throw new FormatException($"Input '{value}' is not a valid Package URL."); + } + + return parsed!; + } + + public string ToCanonicalString() + { + var builder = new StringBuilder("pkg:"); + builder.Append(Type); + builder.Append('/'); + + if (!NamespaceSegments.IsDefaultOrEmpty) + { + builder.Append(string.Join('/', NamespaceSegments)); + builder.Append('/'); + } + + builder.Append(Name); + + if (!string.IsNullOrEmpty(Version)) + { + builder.Append('@'); + builder.Append(Version); + } + + if (!Qualifiers.IsDefaultOrEmpty && Qualifiers.Length > 0) + { + builder.Append('?'); + builder.Append(string.Join('&', Qualifiers.Select(static kvp => $"{kvp.Key}={kvp.Value}"))); + } + + if (!SubpathSegments.IsDefaultOrEmpty && SubpathSegments.Length > 0) + { + builder.Append('#'); + builder.Append(string.Join('/', SubpathSegments)); + } + + return builder.ToString(); + } + + public override string ToString() => ToCanonicalString(); + + private static ImmutableArray> ParseQualifiers(string qualifierPart) + { + if (string.IsNullOrEmpty(qualifierPart)) + { + return ImmutableArray>.Empty; + } + + var entries = qualifierPart.Split('&', StringSplitOptions.RemoveEmptyEntries); + var map = new SortedDictionary(StringComparer.Ordinal); + foreach (var entry in entries) + { + var trimmed = entry.Trim(); + if (trimmed.Length == 0) + { + continue; + } + + var equalsIndex = trimmed.IndexOf('='); + if (equalsIndex <= 0) + { + continue; + } + + var key = Uri.UnescapeDataString(trimmed[..equalsIndex]).Trim().ToLowerInvariant(); + var valuePart = equalsIndex < trimmed.Length - 1 ? trimmed[(equalsIndex + 1)..] : string.Empty; + var value = NormalizeComponent(valuePart, escape: true, lowerCase: false); + map[key] = value; + } + + return map.Select(static kvp => new KeyValuePair(kvp.Key, kvp.Value)).ToImmutableArray(); + } + + private static ImmutableArray ParseSubpath(string subpathPart) + { + if (string.IsNullOrEmpty(subpathPart)) + { + return ImmutableArray.Empty; + } + + var segments = subpathPart.Split('/', StringSplitOptions.RemoveEmptyEntries); + var builder = ImmutableArray.CreateBuilder(segments.Length); + foreach (var raw in segments) + { + var segment = Uri.UnescapeDataString(raw.Trim()); + if (segment.Length == 0) + { + continue; + } + + builder.Add(EscapePathSegment(segment)); + } + + return builder.ToImmutable(); + } + + private static string NormalizeComponent(string? value, bool escape, bool lowerCase) + { + if (string.IsNullOrWhiteSpace(value)) + { + return string.Empty; + } + + var unescaped = Uri.UnescapeDataString(value.Trim()); + if (lowerCase) + { + unescaped = unescaped.ToLowerInvariant(); + } + + return escape ? Uri.EscapeDataString(unescaped) : unescaped; + } + + private static string EscapePathSegment(string value) + { + return Uri.EscapeDataString(value); + } +} diff --git a/src/StellaOps.Feedser.Normalization/SemVer/SemVerRangeRuleBuilder.cs b/src/StellaOps.Feedser.Normalization/SemVer/SemVerRangeRuleBuilder.cs new file mode 100644 index 00000000..d0a8b4b5 --- /dev/null +++ b/src/StellaOps.Feedser.Normalization/SemVer/SemVerRangeRuleBuilder.cs @@ -0,0 +1,615 @@ +using System.Collections.Generic; +using System.Globalization; +using System.Linq; +using System.Text; +using System.Text.RegularExpressions; +using NuGet.Versioning; +using StellaOps.Feedser.Models; + +namespace StellaOps.Feedser.Normalization.SemVer; + +/// +/// Parses SemVer-style range expressions into deterministic primitives and normalized rules. +/// Supports caret, tilde, wildcard, hyphen, and comparator syntaxes with optional provenance notes. +/// +public static class SemVerRangeRuleBuilder +{ + private static readonly Regex ComparatorRegex = new(@"^(?>=|<=|>|<|==|=)\s*(?.+)$", RegexOptions.Compiled | RegexOptions.CultureInvariant); + private static readonly Regex HyphenRegex = new(@"^\s*(?.+?)\s+-\s+(?.+)\s*$", RegexOptions.Compiled | RegexOptions.CultureInvariant); + private static readonly char[] SegmentTrimCharacters = { '(', ')', '[', ']', '{', '}', ';' }; + private static readonly char[] FragmentSplitCharacters = { ',', ' ' }; + + public static IReadOnlyList Build(string? rawRange, string? patchedVersion = null, string? provenanceNote = null) + { + var results = new List(); + + if (!string.IsNullOrWhiteSpace(rawRange)) + { + foreach (var segment in SplitSegments(rawRange)) + { + if (TryBuildFromSegment(segment, patchedVersion, provenanceNote, out var result)) + { + results.Add(result); + } + } + } + + if (results.Count == 0 && TryNormalizeVersion(patchedVersion, out var normalizedPatched)) + { + var constraint = $"< {normalizedPatched}"; + var fallbackPrimitive = new SemVerPrimitive( + Introduced: null, + IntroducedInclusive: false, + Fixed: normalizedPatched, + FixedInclusive: false, + LastAffected: null, + LastAffectedInclusive: false, + ConstraintExpression: constraint); + + var fallbackRule = fallbackPrimitive.ToNormalizedVersionRule(provenanceNote); + if (fallbackRule is not null) + { + results.Add(new SemVerRangeBuildResult(fallbackPrimitive, fallbackRule, constraint)); + } + } + + return results.Count == 0 ? Array.Empty() : results; + } + + private static IEnumerable SplitSegments(string rawRange) + => rawRange.Split("||", StringSplitOptions.RemoveEmptyEntries | StringSplitOptions.TrimEntries); + + private static bool TryBuildFromSegment(string segment, string? patchedVersion, string? provenanceNote, out SemVerRangeBuildResult result) + { + result = default!; + + if (!TryParsePrimitive(segment, patchedVersion, out var primitive)) + { + return false; + } + + var normalized = primitive.ToNormalizedVersionRule(provenanceNote); + if (normalized is null) + { + return false; + } + + result = new SemVerRangeBuildResult(primitive, normalized, primitive.ConstraintExpression ?? segment.Trim()); + return true; + } + + private static bool TryParsePrimitive(string segment, string? patchedVersion, out SemVerPrimitive primitive) + { + primitive = default!; + + var trimmed = NormalizeWhitespace(segment); + if (string.IsNullOrEmpty(trimmed) || IsWildcardToken(trimmed)) + { + return false; + } + + if (TryParseCaret(trimmed, out primitive)) + { + return true; + } + + if (TryParseTilde(trimmed, out primitive)) + { + return true; + } + + if (TryParseHyphen(trimmed, out primitive)) + { + return true; + } + + if (TryParseWildcard(trimmed, out primitive)) + { + return true; + } + + if (TryParseComparators(trimmed, patchedVersion, out primitive)) + { + return true; + } + + if (TryParseExact(trimmed, out primitive)) + { + return true; + } + + return false; + } + + private static bool TryParseCaret(string expression, out SemVerPrimitive primitive) + { + primitive = default!; + + if (!expression.StartsWith("^", StringComparison.Ordinal)) + { + return false; + } + + var value = expression[1..].Trim(); + if (!TryParseSemanticVersion(value, out var version, out var normalizedBase)) + { + return false; + } + + var upper = CalculateCaretUpperBound(version); + var upperNormalized = FormatVersion(upper); + var constraint = $">= {normalizedBase} < {upperNormalized}"; + + primitive = new SemVerPrimitive( + Introduced: normalizedBase, + IntroducedInclusive: true, + Fixed: upperNormalized, + FixedInclusive: false, + LastAffected: null, + LastAffectedInclusive: false, + ConstraintExpression: constraint); + return true; + } + + private static bool TryParseTilde(string expression, out SemVerPrimitive primitive) + { + primitive = default!; + + if (!expression.StartsWith("~", StringComparison.Ordinal)) + { + return false; + } + + var remainder = expression.TrimStart('~'); + if (remainder.StartsWith(">", StringComparison.Ordinal)) + { + remainder = remainder[1..]; + } + + remainder = remainder.Trim(); + if (!TryParseSemanticVersion(remainder, out var version, out var normalizedBase)) + { + return false; + } + + var componentCount = CountExplicitComponents(remainder); + var upper = CalculateTildeUpperBound(version, componentCount); + var upperNormalized = FormatVersion(upper); + var constraint = $">= {normalizedBase} < {upperNormalized}"; + + primitive = new SemVerPrimitive( + Introduced: normalizedBase, + IntroducedInclusive: true, + Fixed: upperNormalized, + FixedInclusive: false, + LastAffected: null, + LastAffectedInclusive: false, + ConstraintExpression: constraint); + return true; + } + + private static bool TryParseHyphen(string expression, out SemVerPrimitive primitive) + { + primitive = default!; + + var match = HyphenRegex.Match(expression); + if (!match.Success) + { + return false; + } + + var startRaw = match.Groups["start"].Value.Trim(); + var endRaw = match.Groups["end"].Value.Trim(); + + if (!TryNormalizeVersion(startRaw, out var start) || !TryNormalizeVersion(endRaw, out var end)) + { + return false; + } + + var constraint = $"{start} - {end}"; + primitive = new SemVerPrimitive( + Introduced: start, + IntroducedInclusive: true, + Fixed: null, + FixedInclusive: false, + LastAffected: end, + LastAffectedInclusive: true, + ConstraintExpression: constraint); + return true; + } + + private static bool TryParseWildcard(string expression, out SemVerPrimitive primitive) + { + primitive = default!; + + var sanitized = expression.Trim(SegmentTrimCharacters).Trim(); + if (string.IsNullOrEmpty(sanitized)) + { + return false; + } + + var parts = sanitized.Split('.', StringSplitOptions.RemoveEmptyEntries | StringSplitOptions.TrimEntries); + if (parts.Length == 0 || parts.All(static part => !IsWildcardToken(part))) + { + return false; + } + + if (!int.TryParse(RemoveLeadingV(parts[0]), NumberStyles.Integer, CultureInfo.InvariantCulture, out var major)) + { + return false; + } + + var hasMinor = parts.Length > 1 && !IsWildcardToken(parts[1]); + var hasPatch = parts.Length > 2 && !IsWildcardToken(parts[2]); + var minor = hasMinor && int.TryParse(parts[1], NumberStyles.Integer, CultureInfo.InvariantCulture, out var parsedMinor) ? parsedMinor : 0; + var patch = hasPatch && int.TryParse(parts[2], NumberStyles.Integer, CultureInfo.InvariantCulture, out var parsedPatch) ? parsedPatch : 0; + + var minVersion = new SemanticVersion(major, hasMinor ? minor : 0, hasPatch ? patch : 0); + var minNormalized = FormatVersion(minVersion); + + SemanticVersion upperVersion; + if (!hasMinor) + { + upperVersion = new SemanticVersion(major + 1, 0, 0); + } + else if (!hasPatch) + { + upperVersion = new SemanticVersion(major, minor + 1, 0); + } + else + { + upperVersion = new SemanticVersion(major, minor + 1, 0); + } + + var upperNormalized = FormatVersion(upperVersion); + var constraint = $">= {minNormalized} < {upperNormalized}"; + + primitive = new SemVerPrimitive( + Introduced: minNormalized, + IntroducedInclusive: true, + Fixed: upperNormalized, + FixedInclusive: false, + LastAffected: null, + LastAffectedInclusive: false, + ConstraintExpression: constraint); + return true; + } + + private static bool TryParseComparators(string expression, string? patchedVersion, out SemVerPrimitive primitive) + { + primitive = default!; + + var fragments = expression.Split(',', StringSplitOptions.RemoveEmptyEntries | StringSplitOptions.TrimEntries); + if (fragments.Length == 0) + { + fragments = new[] { expression }; + } + + var constraintTokens = new List(); + + string? introduced = null; + bool introducedInclusive = true; + var hasIntroduced = false; + + string? fixedVersion = null; + bool fixedInclusive = false; + var hasFixed = false; + + string? lastAffected = null; + bool lastInclusive = true; + var hasLast = false; + + string? exactValue = null; + + foreach (var fragment in fragments) + { + if (TryParseComparatorFragment(fragment, constraintTokens, ref introduced, ref introducedInclusive, ref hasIntroduced, ref fixedVersion, ref fixedInclusive, ref hasFixed, ref lastAffected, ref lastInclusive, ref hasLast, ref exactValue)) + { + continue; + } + + var parts = fragment.Split(FragmentSplitCharacters, StringSplitOptions.RemoveEmptyEntries | StringSplitOptions.TrimEntries); + var handled = false; + foreach (var part in parts) + { + if (TryParseComparatorFragment(part, constraintTokens, ref introduced, ref introducedInclusive, ref hasIntroduced, ref fixedVersion, ref fixedInclusive, ref hasFixed, ref lastAffected, ref lastInclusive, ref hasLast, ref exactValue)) + { + handled = true; + } + } + + if (handled) + { + continue; + } + + if (TryNormalizeVersion(fragment, out var normalizedExact)) + { + exactValue = normalizedExact; + constraintTokens.Add(normalizedExact); + } + } + + if (exactValue is not null) + { + primitive = new SemVerPrimitive( + Introduced: exactValue, + IntroducedInclusive: true, + Fixed: exactValue, + FixedInclusive: true, + LastAffected: null, + LastAffectedInclusive: false, + ConstraintExpression: constraintTokens.Count > 0 ? string.Join(", ", constraintTokens) : expression.Trim()); + return true; + } + + if (hasIntroduced && !hasFixed && !hasLast && TryNormalizeVersion(patchedVersion, out var patchedNormalized)) + { + fixedVersion = patchedNormalized; + fixedInclusive = false; + hasFixed = true; + } + + if (!hasIntroduced && !hasFixed && !hasLast) + { + primitive = default!; + return false; + } + + var constraint = constraintTokens.Count > 0 ? string.Join(", ", constraintTokens) : expression.Trim(); + primitive = new SemVerPrimitive( + Introduced: hasIntroduced ? introduced : null, + IntroducedInclusive: hasIntroduced ? introducedInclusive : true, + Fixed: hasFixed ? fixedVersion : null, + FixedInclusive: hasFixed ? fixedInclusive : false, + LastAffected: hasLast ? lastAffected : null, + LastAffectedInclusive: hasLast ? lastInclusive : false, + ConstraintExpression: constraint); + return true; + } + + private static bool TryParseExact(string expression, out SemVerPrimitive primitive) + { + primitive = default!; + + if (!TryNormalizeVersion(expression, out var normalized)) + { + return false; + } + + primitive = new SemVerPrimitive( + Introduced: normalized, + IntroducedInclusive: true, + Fixed: normalized, + FixedInclusive: true, + LastAffected: null, + LastAffectedInclusive: false, + ConstraintExpression: normalized); + return true; + } + + private static bool TryParseComparatorFragment( + string fragment, + ICollection constraintTokens, + ref string? introduced, + ref bool introducedInclusive, + ref bool hasIntroduced, + ref string? fixedVersion, + ref bool fixedInclusive, + ref bool hasFixed, + ref string? lastAffected, + ref bool lastInclusive, + ref bool hasLast, + ref string? exactValue) + { + var trimmed = fragment.Trim(); + trimmed = trimmed.Trim(SegmentTrimCharacters); + + if (string.IsNullOrEmpty(trimmed)) + { + return false; + } + + var match = ComparatorRegex.Match(trimmed); + if (!match.Success) + { + return false; + } + + var op = match.Groups["op"].Value; + var rawValue = match.Groups["value"].Value; + if (!TryNormalizeVersion(rawValue, out var value)) + { + return true; + } + + switch (op) + { + case ">=": + introduced = value; + introducedInclusive = true; + hasIntroduced = true; + constraintTokens.Add($">= {value}"); + break; + case ">": + introduced = value; + introducedInclusive = false; + hasIntroduced = true; + constraintTokens.Add($"> {value}"); + break; + case "<=": + lastAffected = value; + lastInclusive = true; + hasLast = true; + constraintTokens.Add($"<= {value}"); + break; + case "<": + fixedVersion = value; + fixedInclusive = false; + hasFixed = true; + constraintTokens.Add($"< {value}"); + break; + case "=": + case "==": + exactValue = value; + constraintTokens.Add($"= {value}"); + break; + } + + return true; + } + + private static bool TryNormalizeVersion(string? value, out string normalized) + { + normalized = string.Empty; + if (string.IsNullOrWhiteSpace(value)) + { + return false; + } + + var trimmed = value.Trim(); + trimmed = trimmed.Trim(SegmentTrimCharacters); + trimmed = trimmed.Trim('\'', '"', '`'); + + if (string.IsNullOrWhiteSpace(trimmed) || IsWildcardToken(trimmed)) + { + return false; + } + + var candidate = RemoveLeadingV(trimmed); + if (SemanticVersion.TryParse(candidate, out var semanticVersion)) + { + normalized = FormatVersion(semanticVersion); + return true; + } + + if (trimmed.IndexOfAny(new[] { '*', 'x', 'X' }) >= 0) + { + return false; + } + + normalized = candidate; + return true; + } + + private static bool TryParseSemanticVersion(string value, out SemanticVersion version, out string normalized) + { + version = null!; + normalized = string.Empty; + + var candidate = RemoveLeadingV(value); + if (!SemanticVersion.TryParse(candidate, out version)) + { + candidate = ExpandSemanticVersion(candidate); + if (!SemanticVersion.TryParse(candidate, out version)) + { + return false; + } + } + + normalized = FormatVersion(version); + return true; + } + + private static string ExpandSemanticVersion(string value) + { + var partCount = value.Count(static ch => ch == '.'); + return partCount switch + { + 0 => value + ".0.0", + 1 => value + ".0", + _ => value, + }; + } + + private static SemanticVersion CalculateCaretUpperBound(SemanticVersion baseVersion) + { + if (baseVersion.Major > 0) + { + return new SemanticVersion(baseVersion.Major + 1, 0, 0); + } + + if (baseVersion.Minor > 0) + { + return new SemanticVersion(0, baseVersion.Minor + 1, 0); + } + + return new SemanticVersion(0, 0, baseVersion.Patch + 1); + } + + private static SemanticVersion CalculateTildeUpperBound(SemanticVersion baseVersion, int componentCount) + { + if (componentCount <= 1) + { + return new SemanticVersion(baseVersion.Major + 1, 0, 0); + } + + return new SemanticVersion(baseVersion.Major, baseVersion.Minor + 1, 0); + } + + private static int CountExplicitComponents(string value) + { + var head = value.Split(new[] { '-', '+' }, 2, StringSplitOptions.RemoveEmptyEntries | StringSplitOptions.TrimEntries); + return head.Length == 0 + ? 0 + : head[0].Split('.', StringSplitOptions.RemoveEmptyEntries | StringSplitOptions.TrimEntries).Length; + } + + private static string NormalizeWhitespace(string value) + { + var builder = new StringBuilder(value.Length); + var previousWhitespace = false; + + foreach (var ch in value) + { + if (char.IsWhiteSpace(ch)) + { + if (!previousWhitespace) + { + builder.Append(' '); + previousWhitespace = true; + } + } + else + { + builder.Append(ch); + previousWhitespace = false; + } + } + + return builder.ToString().Trim(); + } + + private static string RemoveLeadingV(string value) + { + if (value.Length > 1 && (value[0] == 'v' || value[0] == 'V') && char.IsDigit(value[1])) + { + return value[1..]; + } + + return value; + } + + private static string FormatVersion(SemanticVersion version) + { + var normalized = version.ToNormalizedString(); + if (!string.IsNullOrEmpty(version.Metadata)) + { + normalized += "+" + version.Metadata; + } + + return normalized; + } + + private static bool IsWildcardToken(string value) + => string.Equals(value, "*", StringComparison.OrdinalIgnoreCase) + || string.Equals(value, "x", StringComparison.OrdinalIgnoreCase); +} + +public sealed record SemVerRangeBuildResult( + SemVerPrimitive Primitive, + NormalizedVersionRule NormalizedRule, + string Expression) +{ + public string? ConstraintExpression => Primitive.ConstraintExpression; +} diff --git a/src/StellaOps.Feedser.Normalization/StellaOps.Feedser.Normalization.csproj b/src/StellaOps.Feedser.Normalization/StellaOps.Feedser.Normalization.csproj new file mode 100644 index 00000000..203716ed --- /dev/null +++ b/src/StellaOps.Feedser.Normalization/StellaOps.Feedser.Normalization.csproj @@ -0,0 +1,22 @@ + + + + net10.0 + enable + enable + false + + + + + + + + + + + + + + + diff --git a/src/StellaOps.Feedser.Normalization/TASKS.md b/src/StellaOps.Feedser.Normalization/TASKS.md new file mode 100644 index 00000000..8d547682 --- /dev/null +++ b/src/StellaOps.Feedser.Normalization/TASKS.md @@ -0,0 +1,8 @@ +# TASKS +| Task | Owner(s) | Depends on | Notes | +|---|---|---|---| +|Canonical NEVRA/EVR parsing helpers|BE-Norm (Distro WG)|Models|DONE – `Normalization.Distro` exposes parsers + canonical formatters consumed by Merge comparers/tests.| +|PURL/CPE identifier normalization|BE-Norm (OSS WG)|Models|DONE – canonical PURL/CPE helpers feed connectors and exporter tooling.| +|CPE normalization escape handling|BE-Norm (OSS WG)|Normalization identifiers|DONE – percent-decoding, edition sub-field expansion, and deterministic escaping landed in `Cpe23` with new tests covering boundary cases.| +|CVSS metric normalization & severity bands|BE-Norm (Risk WG)|Models|DONE – `CvssMetricNormalizer` unifies vectors, recomputes scores/severities, and is wired through NVD/RedHat/JVN mappers with unit coverage.| +|Description and locale normalization pipeline|BE-Norm (I18N)|Source connectors|DONE – `DescriptionNormalizer` strips markup, collapses whitespace, and provides locale fallback used by core mappers.| diff --git a/src/StellaOps.Feedser.Normalization/Text/DescriptionNormalizer.cs b/src/StellaOps.Feedser.Normalization/Text/DescriptionNormalizer.cs new file mode 100644 index 00000000..d43d25a5 --- /dev/null +++ b/src/StellaOps.Feedser.Normalization/Text/DescriptionNormalizer.cs @@ -0,0 +1,118 @@ +using System.Globalization; +using System.Linq; +using System.Net; +using System.Text.RegularExpressions; + +namespace StellaOps.Feedser.Normalization.Text; + +/// +/// Normalizes advisory descriptions by stripping markup, collapsing whitespace, and selecting the best locale fallback. +/// +public static class DescriptionNormalizer +{ + private static readonly Regex HtmlTagRegex = new("<[^>]+>", RegexOptions.Compiled | RegexOptions.CultureInvariant); + private static readonly Regex WhitespaceRegex = new("\\s+", RegexOptions.Compiled | RegexOptions.CultureInvariant); + private static readonly string[] PreferredLanguages = { "en", "en-us", "en-gb" }; + + public static NormalizedDescription Normalize(IEnumerable candidates) + { + if (candidates is null) + { + throw new ArgumentNullException(nameof(candidates)); + } + + var processed = new List<(string Text, string Language, int Index)>(); + var index = 0; + foreach (var candidate in candidates) + { + if (string.IsNullOrWhiteSpace(candidate.Text)) + { + index++; + continue; + } + + var sanitized = Sanitize(candidate.Text); + if (string.IsNullOrWhiteSpace(sanitized)) + { + index++; + continue; + } + + var language = NormalizeLanguage(candidate.Language); + processed.Add((sanitized, language, index)); + index++; + } + + if (processed.Count == 0) + { + return new NormalizedDescription(string.Empty, "en"); + } + + var best = SelectBest(processed); + var languageTag = best.Language.Length > 0 ? best.Language : "en"; + return new NormalizedDescription(best.Text, languageTag); + } + + private static (string Text, string Language) SelectBest(List<(string Text, string Language, int Index)> processed) + { + foreach (var preferred in PreferredLanguages) + { + var normalized = NormalizeLanguage(preferred); + var match = processed.FirstOrDefault(entry => entry.Language.Equals(normalized, StringComparison.OrdinalIgnoreCase)); + if (!string.IsNullOrEmpty(match.Text)) + { + return (match.Text, normalized); + } + } + + var first = processed.OrderBy(entry => entry.Index).First(); + return (first.Text, first.Language); + } + + private static string Sanitize(string text) + { + var decoded = WebUtility.HtmlDecode(text) ?? string.Empty; + var withoutTags = HtmlTagRegex.Replace(decoded, " "); + var collapsed = WhitespaceRegex.Replace(withoutTags, " ").Trim(); + return collapsed; + } + + private static string NormalizeLanguage(string? language) + { + if (string.IsNullOrWhiteSpace(language)) + { + return string.Empty; + } + + var trimmed = language.Trim(); + try + { + var culture = CultureInfo.GetCultureInfo(trimmed); + if (!string.IsNullOrEmpty(culture.Name)) + { + var parts = culture.Name.Split('-'); + if (parts.Length > 0 && !string.IsNullOrWhiteSpace(parts[0])) + { + return parts[0].ToLowerInvariant(); + } + } + } + catch (CultureNotFoundException) + { + // fall back to manual normalization + } + + var primary = trimmed.Split(new[] { '-', '_' }, StringSplitOptions.RemoveEmptyEntries).FirstOrDefault(); + return string.IsNullOrWhiteSpace(primary) ? string.Empty : primary.ToLowerInvariant(); + } +} + +/// +/// Represents a localized text candidate. +/// +public readonly record struct LocalizedText(string? Text, string? Language); + +/// +/// Represents a normalized description result. +/// +public readonly record struct NormalizedDescription(string Text, string Language); diff --git a/src/StellaOps.Feedser.Source.Acsc/AGENTS.md b/src/StellaOps.Feedser.Source.Acsc/AGENTS.md new file mode 100644 index 00000000..c090d238 --- /dev/null +++ b/src/StellaOps.Feedser.Source.Acsc/AGENTS.md @@ -0,0 +1,40 @@ +# AGENTS +## Role +Bootstrap the ACSC (Australian Cyber Security Centre) advisories connector so the Feedser pipeline can ingest, normalise, and enrich ACSC security bulletins. + +## Scope +- Research the authoritative ACSC advisory feed (RSS/Atom, JSON API, or HTML). +- Implement fetch windowing, cursor persistence, and retry strategy consistent with other external connectors. +- Parse advisory content (summary, affected products, mitigation guidance, references). +- Map advisories into canonical `Advisory` records with aliases, references, affected packages, and provenance metadata. +- Provide deterministic fixtures and regression tests that cover fetch/parse/map flows. + +## Participants +- `Source.Common` for HTTP client creation, fetch service, and DTO persistence helpers. +- `Storage.Mongo` for raw/document/DTO/advisory storage plus cursor management. +- `Feedser.Models` for canonical advisory structures and provenance utilities. +- `Feedser.Testing` for integration harnesses and snapshot helpers. + +## Interfaces & Contracts +- Job kinds should follow the pattern `acsc:fetch`, `acsc:parse`, `acsc:map`. +- Documents persisted to Mongo must include ETag/Last-Modified metadata when the source exposes it. +- Canonical advisories must emit aliases (ACSC ID + CVE IDs) and references (official bulletin + vendor notices). + +## In/Out of scope +In scope: +- Initial end-to-end connector implementation with tests, fixtures, and range primitive coverage. +- Minimal telemetry (logging + diagnostics counters) consistent with other connectors. + +Out of scope: +- Upstream remediation automation or vendor-specific enrichment beyond ACSC data. +- Export-related changes (handled by exporter teams). + +## Observability & Security Expectations +- Log key lifecycle events (fetch/page processed, parse success/error counts, mapping stats). +- Sanitise HTML safely and avoid persisting external scripts or embedded media. +- Handle transient fetch failures gracefully with exponential backoff and mark failures in source state. + +## Tests +- Add integration-style tests under `StellaOps.Feedser.Source.Acsc.Tests` covering fetch/parse/map with canned fixtures. +- Snapshot canonical advisories; provide UPDATE flag flow for regeneration. +- Validate determinism (ordering, casing, timestamps) to satisfy pipeline reproducibility requirements. diff --git a/src/StellaOps.Feedser.Source.Acsc/Class1.cs b/src/StellaOps.Feedser.Source.Acsc/Class1.cs new file mode 100644 index 00000000..435b2bcd --- /dev/null +++ b/src/StellaOps.Feedser.Source.Acsc/Class1.cs @@ -0,0 +1,29 @@ +using System; +using System.Threading; +using System.Threading.Tasks; +using StellaOps.Plugin; + +namespace StellaOps.Feedser.Source.Acsc; + +public sealed class AcscConnectorPlugin : IConnectorPlugin +{ + public string Name => "acsc"; + + public bool IsAvailable(IServiceProvider services) => true; + + public IFeedConnector Create(IServiceProvider services) => new StubConnector(Name); + + private sealed class StubConnector : IFeedConnector + { + public StubConnector(string sourceName) => SourceName = sourceName; + + public string SourceName { get; } + + public Task FetchAsync(IServiceProvider services, CancellationToken cancellationToken) => Task.CompletedTask; + + public Task ParseAsync(IServiceProvider services, CancellationToken cancellationToken) => Task.CompletedTask; + + public Task MapAsync(IServiceProvider services, CancellationToken cancellationToken) => Task.CompletedTask; + } +} + diff --git a/src/StellaOps.Feedser.Source.Acsc/StellaOps.Feedser.Source.Acsc.csproj b/src/StellaOps.Feedser.Source.Acsc/StellaOps.Feedser.Source.Acsc.csproj new file mode 100644 index 00000000..f7f2c154 --- /dev/null +++ b/src/StellaOps.Feedser.Source.Acsc/StellaOps.Feedser.Source.Acsc.csproj @@ -0,0 +1,16 @@ + + + + net10.0 + enable + enable + + + + + + + + + + diff --git a/src/StellaOps.Feedser.Source.Acsc/TASKS.md b/src/StellaOps.Feedser.Source.Acsc/TASKS.md new file mode 100644 index 00000000..7b10e977 --- /dev/null +++ b/src/StellaOps.Feedser.Source.Acsc/TASKS.md @@ -0,0 +1,11 @@ +# TASKS +| Task | Owner(s) | Depends on | Notes | +|---|---|---|---| +|FEEDCONN-ACSC-02-001 Source discovery & feed contract|BE-Conn-ACSC|Research|**DOING** – 2025-10-11: Catalogued cyber.gov.au RSS endpoints (alerts/advisories/news/publications/threats); HTTPS pulls hit HTTP/2 INTERNAL_ERROR but HTTP fallback via relay works, so we must document history window, caching headers, and partner portal deltas before closing.| +|FEEDCONN-ACSC-02-002 Fetch pipeline & cursor persistence|BE-Conn-ACSC|Source.Common, Storage.Mongo|**TODO** – Implement HTTP client registration, fetch jobs, duplicate detection, and cursor storage (last published + pending docs).| +|FEEDCONN-ACSC-02-003 Parser & DTO sanitiser|BE-Conn-ACSC|Source.Common|**TODO** – Build DTOs for advisory detail content, extract summary/description/affected products/references, and normalise HTML safely.| +|FEEDCONN-ACSC-02-004 Canonical mapper + range primitives|BE-Conn-ACSC|Models|**TODO** – Map advisories to canonical `Advisory` objects with aliases, references, affected packages, and vendor `RangePrimitives`. Coordinate SemVer rule output per `../StellaOps.Feedser.Merge/RANGE_PRIMITIVES_COORDINATION.md`.| +|FEEDCONN-ACSC-02-005 Deterministic fixtures & regression tests|QA|Testing|**TODO** – Add fetch/parse/map regression tests with canned fixtures; support `UPDATE_ACSC_FIXTURES=1` for regeneration.| +|FEEDCONN-ACSC-02-006 Diagnostics & documentation|DevEx|Docs|**TODO** – Document connector behaviour in module README, add observability counters/logging, and update backlog once complete.| +|FEEDCONN-ACSC-02-007 Feed retention & pagination validation|BE-Conn-ACSC|Research|**DOING** – 2025-10-11: HTTP relay confirms feed contains August/July 2025 advisories; need to measure window depth, inspect caching headers, and outline scrape fallback if retention is limited.| +|FEEDCONN-ACSC-02-008 HTTP client compatibility plan|BE-Conn-ACSC|Source.Common|**TODO** – Evaluate proxy/TLS options to bypass Akamai HTTP/2 INTERNAL_ERROR responses, document required client headers, and surface retry/backoff guidance for fetch jobs.| diff --git a/src/StellaOps.Feedser.Source.Cccs/AGENTS.md b/src/StellaOps.Feedser.Source.Cccs/AGENTS.md new file mode 100644 index 00000000..f64aaef7 --- /dev/null +++ b/src/StellaOps.Feedser.Source.Cccs/AGENTS.md @@ -0,0 +1,40 @@ +# AGENTS +## Role +Build the CCCS (Canadian Centre for Cyber Security) advisories connector so Feedser can ingest national cyber bulletins alongside other vendor/regional sources. + +## Scope +- Research CCCS advisory feeds (RSS/Atom, JSON API, or HTML listings) and define the canonical fetch workflow. +- Implement fetch, parse, and mapping stages with deterministic cursoring and retry/backoff behaviour. +- Normalise advisory content (summary, affected vendors/products, mitigation guidance, references, CVE IDs). +- Emit canonical `Advisory` records with aliases, references, affected packages, and provenance metadata. +- Provide fixtures and regression tests to keep the connector deterministic. + +## Participants +- `Source.Common` (HTTP clients, fetch service, DTO storage helpers). +- `Storage.Mongo` (raw/document/DTO/advisory stores + source state). +- `Feedser.Models` (canonical advisory data structures). +- `Feedser.Testing` (integration fixtures and snapshot utilities). + +## Interfaces & Contracts +- Job kinds: `cccs:fetch`, `cccs:parse`, `cccs:map`. +- Persist ETag/Last-Modified metadata when the upstream supports it. +- Include alias entries for CCCS advisory IDs plus referenced CVE IDs. + +## In/Out of scope +In scope: +- End-to-end connector implementation with range primitive coverage for affected packages. +- Minimal telemetry logging/counters matching other connectors. + +Out of scope: +- Automated remediation actions or vendor-specific enrichment beyond CCCS published data. +- Export or downstream pipeline changes. + +## Observability & Security Expectations +- Log fetch attempts, success/failure counts, and mapping statistics. +- Sanitize HTML safely, dropping scripts/styles before storing DTOs. +- Respect upstream rate limits; mark failures in source state with backoff. + +## Tests +- Add `StellaOps.Feedser.Source.Cccs.Tests` covering fetch/parse/map with canned fixtures. +- Snapshot canonical advisories; support fixture regeneration via env flag. +- Validate deterministic ordering and timestamps to maintain reproducibility. diff --git a/src/StellaOps.Feedser.Source.Cccs/Class1.cs b/src/StellaOps.Feedser.Source.Cccs/Class1.cs new file mode 100644 index 00000000..220d4c88 --- /dev/null +++ b/src/StellaOps.Feedser.Source.Cccs/Class1.cs @@ -0,0 +1,29 @@ +using System; +using System.Threading; +using System.Threading.Tasks; +using StellaOps.Plugin; + +namespace StellaOps.Feedser.Source.Cccs; + +public sealed class CccsConnectorPlugin : IConnectorPlugin +{ + public string Name => "cccs"; + + public bool IsAvailable(IServiceProvider services) => true; + + public IFeedConnector Create(IServiceProvider services) => new StubConnector(Name); + + private sealed class StubConnector : IFeedConnector + { + public StubConnector(string sourceName) => SourceName = sourceName; + + public string SourceName { get; } + + public Task FetchAsync(IServiceProvider services, CancellationToken cancellationToken) => Task.CompletedTask; + + public Task ParseAsync(IServiceProvider services, CancellationToken cancellationToken) => Task.CompletedTask; + + public Task MapAsync(IServiceProvider services, CancellationToken cancellationToken) => Task.CompletedTask; + } +} + diff --git a/src/StellaOps.Feedser.Source.Cccs/StellaOps.Feedser.Source.Cccs.csproj b/src/StellaOps.Feedser.Source.Cccs/StellaOps.Feedser.Source.Cccs.csproj new file mode 100644 index 00000000..f7f2c154 --- /dev/null +++ b/src/StellaOps.Feedser.Source.Cccs/StellaOps.Feedser.Source.Cccs.csproj @@ -0,0 +1,16 @@ + + + + net10.0 + enable + enable + + + + + + + + + + diff --git a/src/StellaOps.Feedser.Source.Cccs/TASKS.md b/src/StellaOps.Feedser.Source.Cccs/TASKS.md new file mode 100644 index 00000000..2cf48f7d --- /dev/null +++ b/src/StellaOps.Feedser.Source.Cccs/TASKS.md @@ -0,0 +1,10 @@ +# TASKS +| Task | Owner(s) | Depends on | Notes | +|---|---|---|---| +|FEEDCONN-CCCS-02-001 Catalogue official CCCS advisory feeds|BE-Conn-CCCS|Research|**DOING** – 2025-10-11: Confirmed Alerts & Advisories Atom feed (`https://cyber.gc.ca/api/cccs/rss/v1/get?feed=alerts_advisories&lang=en`) delivers full HTML payloads; need to enumerate related feeds (bulletins, vulnerabilities), test history window, and capture caching headers for cursor design.| +|FEEDCONN-CCCS-02-002 Implement fetch & source state handling|BE-Conn-CCCS|Source.Common, Storage.Mongo|**TODO** – Register HTTP client, implement fetch job, persist raw documents with cursor/backoff logic.| +|FEEDCONN-CCCS-02-003 DTO/parser implementation|BE-Conn-CCCS|Source.Common|**TODO** – Define DTOs for CCCS advisories, sanitise HTML/JSON, extract summary, references, CVE lists, and mitigation guidance.| +|FEEDCONN-CCCS-02-004 Canonical mapping & range primitives|BE-Conn-CCCS|Models|**TODO** – Map advisories into canonical records with aliases, references, vendor/package range primitives, and provenance. Align normalized SemVer rules per `../StellaOps.Feedser.Merge/RANGE_PRIMITIVES_COORDINATION.md`.| +|FEEDCONN-CCCS-02-005 Deterministic fixtures & tests|QA|Testing|**TODO** – Add regression tests with canned fixtures; support `UPDATE_CCCS_FIXTURES=1` to refresh snapshots.| +|FEEDCONN-CCCS-02-006 Observability & documentation|DevEx|Docs|**TODO** – Document connector configuration, add logging/metrics, and update backlog once feature-complete.| +|FEEDCONN-CCCS-02-007 Historical advisory harvesting plan|BE-Conn-CCCS|Research|**TODO** – Determine archive access (CSV/HTML), required localization, and plan backfill window for initial import.| diff --git a/src/StellaOps.Feedser.Source.CertBund/AGENTS.md b/src/StellaOps.Feedser.Source.CertBund/AGENTS.md new file mode 100644 index 00000000..724b2fd5 --- /dev/null +++ b/src/StellaOps.Feedser.Source.CertBund/AGENTS.md @@ -0,0 +1,40 @@ +# AGENTS +## Role +Deliver a connector for Germany’s CERT-Bund advisories so Feedser can ingest, normalise, and enrich BSI alerts alongside other national feeds. + +## Scope +- Identify the authoritative CERT-Bund advisory feed(s) (RSS/Atom, JSON, CSV, or HTML). +- Implement fetch/cursor logic with proper windowing, dedupe, and failure backoff. +- Parse advisory detail pages for summary, affected products/vendors, mitigation, and references. +- Map advisories into canonical `Advisory` objects including aliases, references, affected packages, and provenance/range primitives. +- Provide deterministic fixtures and regression tests. + +## Participants +- `Source.Common` (HTTP/fetch utilities, DTO storage). +- `Storage.Mongo` (raw/document/DTO/advisory stores, source state). +- `Feedser.Models` (canonical data model). +- `Feedser.Testing` (integration harness, snapshot utilities). + +## Interfaces & Contracts +- Job kinds: `certbund:fetch`, `certbund:parse`, `certbund:map`. +- Persist upstream metadata (ETag/Last-Modified) if provided. +- Alias set should include CERT-Bund ID and referenced CVE entries. + +## In/Out of scope +In scope: +- End-to-end connector implementation with deterministic tests and range primitive coverage. +- Baseline logging/metrics for pipeline observability. + +Out of scope: +- Non-advisory CERT-Bund digests or newsletters. +- Downstream exporter changes. + +## Observability & Security Expectations +- Log fetch attempts, item counts, and mapping metrics. +- Sanitize HTML thoroughly before persistence. +- Handle transient failures gracefully with exponential backoff and failure records in source state. + +## Tests +- Add `StellaOps.Feedser.Source.CertBund.Tests` covering fetch/parse/map with canned fixtures. +- Snapshot canonical advisories; support regeneration via environment flag. +- Ensure deterministic ordering, casing, and timestamps. diff --git a/src/StellaOps.Feedser.Source.CertBund/Class1.cs b/src/StellaOps.Feedser.Source.CertBund/Class1.cs new file mode 100644 index 00000000..2eb03580 --- /dev/null +++ b/src/StellaOps.Feedser.Source.CertBund/Class1.cs @@ -0,0 +1,29 @@ +using System; +using System.Threading; +using System.Threading.Tasks; +using StellaOps.Plugin; + +namespace StellaOps.Feedser.Source.CertBund; + +public sealed class CertBundConnectorPlugin : IConnectorPlugin +{ + public string Name => "certbund"; + + public bool IsAvailable(IServiceProvider services) => true; + + public IFeedConnector Create(IServiceProvider services) => new StubConnector(Name); + + private sealed class StubConnector : IFeedConnector + { + public StubConnector(string sourceName) => SourceName = sourceName; + + public string SourceName { get; } + + public Task FetchAsync(IServiceProvider services, CancellationToken cancellationToken) => Task.CompletedTask; + + public Task ParseAsync(IServiceProvider services, CancellationToken cancellationToken) => Task.CompletedTask; + + public Task MapAsync(IServiceProvider services, CancellationToken cancellationToken) => Task.CompletedTask; + } +} + diff --git a/src/StellaOps.Feedser.Source.CertBund/StellaOps.Feedser.Source.CertBund.csproj b/src/StellaOps.Feedser.Source.CertBund/StellaOps.Feedser.Source.CertBund.csproj new file mode 100644 index 00000000..f7f2c154 --- /dev/null +++ b/src/StellaOps.Feedser.Source.CertBund/StellaOps.Feedser.Source.CertBund.csproj @@ -0,0 +1,16 @@ + + + + net10.0 + enable + enable + + + + + + + + + + diff --git a/src/StellaOps.Feedser.Source.CertBund/TASKS.md b/src/StellaOps.Feedser.Source.CertBund/TASKS.md new file mode 100644 index 00000000..3a72c9f9 --- /dev/null +++ b/src/StellaOps.Feedser.Source.CertBund/TASKS.md @@ -0,0 +1,10 @@ +# TASKS +| Task | Owner(s) | Depends on | Notes | +|---|---|---|---| +|FEEDCONN-CERTBUND-02-001 Research CERT-Bund advisory endpoints|BE-Conn-CERTBUND|Research|**DOING** – 2025-10-11: BSI RSS directory includes CERT-Bund vulnerability warnings feed; outstanding: confirm detail HTML structure, pagination, and archival coverage.| +|FEEDCONN-CERTBUND-02-002 Fetch job & state persistence|BE-Conn-CERTBUND|Source.Common, Storage.Mongo|**TODO** – Configure HTTP client, implement fetch job with paging/window, persist raw documents with cursor/backoff updates.| +|FEEDCONN-CERTBUND-02-003 Parser/DTO implementation|BE-Conn-CERTBUND|Source.Common|**TODO** – Build DTOs for detail pages, sanitise HTML, extract summary, references, CVE IDs, affected products.| +|FEEDCONN-CERTBUND-02-004 Canonical mapping & range primitives|BE-Conn-CERTBUND|Models|**TODO** – Map advisories into canonical records including aliases, references, affected packages, and vendor range primitives. Follow normalized range guidance in `../StellaOps.Feedser.Merge/RANGE_PRIMITIVES_COORDINATION.md`.| +|FEEDCONN-CERTBUND-02-005 Regression fixtures & tests|QA|Testing|**TODO** – Add deterministic fetch/parse/map tests with fixtures; support `UPDATE_CERTBUND_FIXTURES=1`.| +|FEEDCONN-CERTBUND-02-006 Telemetry & documentation|DevEx|Docs|**TODO** – Add logging/metrics, document connector configuration, and update backlog when feature complete.| +|FEEDCONN-CERTBUND-02-007 Feed history & locale assessment|BE-Conn-CERTBUND|Research|**TODO** – Capture RSS retention depth, assess German-language parsing requirements, and outline fallback plan for historical HTML crawl.| diff --git a/src/StellaOps.Feedser.Source.CertCc.Tests/CertCc/CertCcConnectorFetchTests.cs b/src/StellaOps.Feedser.Source.CertCc.Tests/CertCc/CertCcConnectorFetchTests.cs new file mode 100644 index 00000000..aa6b2ade --- /dev/null +++ b/src/StellaOps.Feedser.Source.CertCc.Tests/CertCc/CertCcConnectorFetchTests.cs @@ -0,0 +1,263 @@ +using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Http; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Logging.Abstractions; +using Microsoft.Extensions.Time.Testing; +using MongoDB.Bson; +using StellaOps.Feedser.Source.CertCc; +using StellaOps.Feedser.Source.CertCc.Configuration; +using StellaOps.Feedser.Source.CertCc.Internal; +using StellaOps.Feedser.Source.Common; +using StellaOps.Feedser.Source.Common.Http; +using StellaOps.Feedser.Source.Common.Cursors; +using StellaOps.Feedser.Source.Common.Testing; +using StellaOps.Feedser.Storage.Mongo; +using StellaOps.Feedser.Storage.Mongo.Documents; +using StellaOps.Feedser.Testing; +using Xunit; + +namespace StellaOps.Feedser.Source.CertCc.Tests.CertCc; + +[Collection("mongo-fixture")] +public sealed class CertCcConnectorFetchTests : IAsyncLifetime +{ + private const string TestNoteId = "294418"; + + private readonly MongoIntegrationFixture _fixture; + private readonly FakeTimeProvider _timeProvider; + private readonly CannedHttpMessageHandler _handler; + private ServiceProvider? _serviceProvider; + + public CertCcConnectorFetchTests(MongoIntegrationFixture fixture) + { + _fixture = fixture; + _timeProvider = new FakeTimeProvider(new DateTimeOffset(2025, 10, 11, 8, 0, 0, TimeSpan.Zero)); + _handler = new CannedHttpMessageHandler(); + } + + [Fact(Skip = "Superseded by snapshot regression coverage (FEEDCONN-CERTCC-02-005).")] + public async Task FetchAsync_PersistsSummaryAndDetailDocumentsAndUpdatesCursor() + { + var template = new CertCcOptions + { + BaseApiUri = new Uri("https://www.kb.cert.org/vuls/api/", UriKind.Absolute), + SummaryWindow = new TimeWindowCursorOptions + { + WindowSize = TimeSpan.FromDays(30), + Overlap = TimeSpan.FromDays(5), + InitialBackfill = TimeSpan.FromDays(60), + MinimumWindowSize = TimeSpan.FromDays(1), + }, + MaxMonthlySummaries = 3, + MaxNotesPerFetch = 3, + DetailRequestDelay = TimeSpan.Zero, + }; + + await EnsureServiceProviderAsync(template); + var provider = _serviceProvider!; + + _handler.Clear(); + + var planner = provider.GetRequiredService(); + var plan = planner.CreatePlan(state: null); + Assert.NotEmpty(plan.Requests); + + foreach (var request in plan.Requests) + { + _handler.AddJsonResponse(request.Uri, BuildSummaryPayload()); + } + + RegisterDetailResponses(); + + var connector = provider.GetRequiredService(); + await connector.FetchAsync(provider, CancellationToken.None); + + var documentStore = provider.GetRequiredService(); + foreach (var request in plan.Requests) + { + var record = await documentStore.FindBySourceAndUriAsync(CertCcConnectorPlugin.SourceName, request.Uri.ToString(), CancellationToken.None); + Assert.NotNull(record); + Assert.Equal(DocumentStatuses.PendingParse, record!.Status); + Assert.NotNull(record.Metadata); + Assert.Equal(request.Scope.ToString().ToLowerInvariant(), record.Metadata!["certcc.scope"]); + Assert.Equal(request.Year.ToString("D4"), record.Metadata["certcc.year"]); + if (request.Month.HasValue) + { + Assert.Equal(request.Month.Value.ToString("D2"), record.Metadata["certcc.month"]); + } + else + { + Assert.False(record.Metadata.ContainsKey("certcc.month")); + } + } + + foreach (var uri in EnumerateDetailUris()) + { + var record = await documentStore.FindBySourceAndUriAsync(CertCcConnectorPlugin.SourceName, uri.ToString(), CancellationToken.None); + Assert.NotNull(record); + Assert.Equal(DocumentStatuses.PendingParse, record!.Status); + Assert.NotNull(record.Metadata); + Assert.Equal(TestNoteId, record.Metadata!["certcc.noteId"]); + } + + var stateRepository = provider.GetRequiredService(); + var state = await stateRepository.TryGetAsync(CertCcConnectorPlugin.SourceName, CancellationToken.None); + Assert.NotNull(state); + + BsonValue summaryValue; + Assert.True(state!.Cursor.TryGetValue("summary", out summaryValue)); + var summaryDocument = Assert.IsType(summaryValue); + Assert.True(summaryDocument.TryGetValue("start", out _)); + Assert.True(summaryDocument.TryGetValue("end", out _)); + + var pendingNotesCount = state.Cursor.TryGetValue("pendingNotes", out var pendingNotesValue) + ? pendingNotesValue.AsBsonArray.Count + : 0; + Assert.Equal(0, pendingNotesCount); + + var pendingSummariesCount = state.Cursor.TryGetValue("pendingSummaries", out var pendingSummariesValue) + ? pendingSummariesValue.AsBsonArray.Count + : 0; + Assert.Equal(0, pendingSummariesCount); + + Assert.True(state.Cursor.TryGetValue("lastRun", out _)); + + Assert.True(_handler.Requests.Count >= plan.Requests.Count); + foreach (var request in _handler.Requests) + { + if (request.Headers.TryGetValue("Accept", out var accept)) + { + Assert.Contains("application/json", accept, StringComparison.OrdinalIgnoreCase); + } + } + } + + private static string BuildSummaryPayload() + { + return $$""" + { + "count": 1, + "notes": [ + "VU#{TestNoteId}" + ] + } + """; + } + + private void RegisterDetailResponses() + { + foreach (var uri in EnumerateDetailUris()) + { + var fixtureName = uri.AbsolutePath.EndsWith("/vendors/", StringComparison.OrdinalIgnoreCase) + ? "vu-294418-vendors.json" + : uri.AbsolutePath.EndsWith("/vuls/", StringComparison.OrdinalIgnoreCase) + ? "vu-294418-vuls.json" + : "vu-294418.json"; + + _handler.AddJsonResponse(uri, ReadFixture(fixtureName)); + } + } + + private static IEnumerable EnumerateDetailUris() + { + var baseUri = new Uri("https://www.kb.cert.org/vuls/api/", UriKind.Absolute); + yield return new Uri(baseUri, $"{TestNoteId}/"); + yield return new Uri(baseUri, $"{TestNoteId}/vendors/"); + yield return new Uri(baseUri, $"{TestNoteId}/vuls/"); + } + + private async Task EnsureServiceProviderAsync(CertCcOptions template) + { + await DisposeServiceProviderAsync(); + await _fixture.Client.DropDatabaseAsync(_fixture.Database.DatabaseNamespace.DatabaseName); + + var services = new ServiceCollection(); + services.AddLogging(builder => builder.AddProvider(NullLoggerProvider.Instance)); + services.AddSingleton(_timeProvider); + services.AddSingleton(_handler); + + services.AddMongoStorage(options => + { + options.ConnectionString = _fixture.Runner.ConnectionString; + options.DatabaseName = _fixture.Database.DatabaseNamespace.DatabaseName; + options.CommandTimeout = TimeSpan.FromSeconds(5); + options.RawDocumentRetention = TimeSpan.Zero; + options.RawDocumentRetentionTtlGrace = TimeSpan.FromMinutes(5); + options.RawDocumentRetentionSweepInterval = TimeSpan.FromHours(1); + }); + + services.AddSourceCommon(); + services.AddCertCcConnector(options => + { + options.BaseApiUri = template.BaseApiUri; + options.SummaryWindow = new TimeWindowCursorOptions + { + WindowSize = template.SummaryWindow.WindowSize, + Overlap = template.SummaryWindow.Overlap, + InitialBackfill = template.SummaryWindow.InitialBackfill, + MinimumWindowSize = template.SummaryWindow.MinimumWindowSize, + }; + options.MaxMonthlySummaries = template.MaxMonthlySummaries; + options.MaxNotesPerFetch = template.MaxNotesPerFetch; + options.DetailRequestDelay = template.DetailRequestDelay; + options.EnableDetailMapping = template.EnableDetailMapping; + }); + + services.Configure(CertCcOptions.HttpClientName, builderOptions => + { + builderOptions.HttpMessageHandlerBuilderActions.Add(builder => builder.PrimaryHandler = _handler); + }); + + _serviceProvider = services.BuildServiceProvider(); + var bootstrapper = _serviceProvider.GetRequiredService(); + await bootstrapper.InitializeAsync(CancellationToken.None); + } + + private async Task DisposeServiceProviderAsync() + { + if (_serviceProvider is null) + { + return; + } + + if (_serviceProvider is IAsyncDisposable asyncDisposable) + { + await asyncDisposable.DisposeAsync(); + } + else + { + _serviceProvider.Dispose(); + } + + _serviceProvider = null; + } + + private static string ReadFixture(string filename) + { + var baseDirectory = AppContext.BaseDirectory; + var primary = Path.Combine(baseDirectory, "Fixtures", filename); + if (File.Exists(primary)) + { + return File.ReadAllText(primary); + } + + return File.ReadAllText(Path.Combine(baseDirectory, filename)); + } + + public Task InitializeAsync() + { + _handler.Clear(); + return Task.CompletedTask; + } + + public async Task DisposeAsync() + { + await DisposeServiceProviderAsync(); + } +} diff --git a/src/StellaOps.Feedser.Source.CertCc.Tests/CertCc/CertCcConnectorSnapshotTests.cs b/src/StellaOps.Feedser.Source.CertCc.Tests/CertCc/CertCcConnectorSnapshotTests.cs new file mode 100644 index 00000000..74b62ba8 --- /dev/null +++ b/src/StellaOps.Feedser.Source.CertCc.Tests/CertCc/CertCcConnectorSnapshotTests.cs @@ -0,0 +1,405 @@ +using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using System.Net; +using System.Net.Http; +using System.Net.Http.Headers; +using System.Text; +using System.Threading; +using System.Threading.Tasks; +using FluentAssertions; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Http; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Logging.Abstractions; +using Microsoft.Extensions.Time.Testing; +using MongoDB.Bson; +using StellaOps.Feedser.Models; +using StellaOps.Feedser.Source.CertCc; +using StellaOps.Feedser.Source.CertCc.Configuration; +using StellaOps.Feedser.Source.Common; +using StellaOps.Feedser.Source.Common.Cursors; +using StellaOps.Feedser.Source.Common.Http; +using StellaOps.Feedser.Source.Common.Testing; +using StellaOps.Feedser.Storage.Mongo; +using StellaOps.Feedser.Storage.Mongo.Documents; +using StellaOps.Feedser.Storage.Mongo.Advisories; +using StellaOps.Feedser.Testing; +using Xunit; + +namespace StellaOps.Feedser.Source.CertCc.Tests.CertCc; + +[Collection("mongo-fixture")] +public sealed class CertCcConnectorSnapshotTests : IAsyncLifetime +{ + private static readonly Uri SeptemberSummaryUri = new("https://www.kb.cert.org/vuls/api/2025/09/summary/"); + private static readonly Uri OctoberSummaryUri = new("https://www.kb.cert.org/vuls/api/2025/10/summary/"); + private static readonly Uri YearlySummaryUri = new("https://www.kb.cert.org/vuls/api/2025/summary/"); + private static readonly Uri NoteDetailUri = new("https://www.kb.cert.org/vuls/api/294418/"); + private static readonly Uri VendorsDetailUri = new("https://www.kb.cert.org/vuls/api/294418/vendors/"); + private static readonly Uri VulsDetailUri = new("https://www.kb.cert.org/vuls/api/294418/vuls/"); + private static readonly Uri VendorStatusesDetailUri = new("https://www.kb.cert.org/vuls/api/294418/vendors/vuls/"); + + private readonly MongoIntegrationFixture _fixture; + private ConnectorTestHarness? _harness; + + public CertCcConnectorSnapshotTests(MongoIntegrationFixture fixture) + { + _fixture = fixture; + } + + [Fact] + public async Task FetchSummaryAndDetails_ProducesDeterministicSnapshots() + { + var initialTime = new DateTimeOffset(2025, 11, 1, 8, 0, 0, TimeSpan.Zero); + var harness = await EnsureHarnessAsync(initialTime); + + RegisterSummaryResponses(harness.Handler); + RegisterDetailResponses(harness.Handler); + + var connector = harness.ServiceProvider.GetRequiredService(); + await connector.FetchAsync(harness.ServiceProvider, CancellationToken.None); + + var documentsSnapshot = await BuildDocumentsSnapshotAsync(harness.ServiceProvider); + WriteOrAssertSnapshot(documentsSnapshot, "certcc-documents.snapshot.json"); + + var stateSnapshot = await BuildStateSnapshotAsync(harness.ServiceProvider); + WriteOrAssertSnapshot(stateSnapshot, "certcc-state.snapshot.json"); + + await connector.ParseAsync(harness.ServiceProvider, CancellationToken.None); + await connector.MapAsync(harness.ServiceProvider, CancellationToken.None); + + var advisoriesSnapshot = await BuildAdvisoriesSnapshotAsync(harness.ServiceProvider); + WriteOrAssertSnapshot(advisoriesSnapshot, "certcc-advisories.snapshot.json"); + + harness.TimeProvider.Advance(TimeSpan.FromMinutes(30)); + RegisterSummaryNotModifiedResponses(harness.Handler); + RegisterDetailNotModifiedResponses(harness.Handler); + + await connector.FetchAsync(harness.ServiceProvider, CancellationToken.None); + harness.Handler.AssertNoPendingResponses(); + + var requestsSnapshot = BuildRequestsSnapshot(harness.Handler.Requests); + WriteOrAssertSnapshot(requestsSnapshot, "certcc-requests.snapshot.json"); + } + + private async Task EnsureHarnessAsync(DateTimeOffset initialTime) + { + if (_harness is not null) + { + return _harness; + } + + var harness = new ConnectorTestHarness(_fixture, initialTime, CertCcOptions.HttpClientName); + await harness.EnsureServiceProviderAsync(services => + { + services.AddLogging(builder => builder.AddProvider(NullLoggerProvider.Instance)); + services.AddCertCcConnector(options => + { + options.BaseApiUri = new Uri("https://www.kb.cert.org/vuls/api/", UriKind.Absolute); + options.SummaryWindow = new TimeWindowCursorOptions + { + WindowSize = TimeSpan.FromDays(30), + Overlap = TimeSpan.FromDays(3), + InitialBackfill = TimeSpan.FromDays(45), + MinimumWindowSize = TimeSpan.FromDays(1), + }; + options.MaxMonthlySummaries = 2; + options.MaxNotesPerFetch = 1; + options.DetailRequestDelay = TimeSpan.Zero; + options.EnableDetailMapping = true; + }); + + services.Configure(CertCcOptions.HttpClientName, options => + { + options.HttpMessageHandlerBuilderActions.Add(builder => builder.PrimaryHandler = harness.Handler); + }); + }); + + _harness = harness; + return harness; + } + + private static async Task BuildDocumentsSnapshotAsync(IServiceProvider provider) + { + var documentStore = provider.GetRequiredService(); + var uris = new[] + { + SeptemberSummaryUri, + OctoberSummaryUri, + new Uri("https://www.kb.cert.org/vuls/api/2025/11/summary/"), + YearlySummaryUri, + NoteDetailUri, + VendorsDetailUri, + VulsDetailUri, + VendorStatusesDetailUri, + }; + + var records = new List(uris.Length); + foreach (var uri in uris) + { + var record = await documentStore.FindBySourceAndUriAsync(CertCcConnectorPlugin.SourceName, uri.ToString(), CancellationToken.None); + if (record is null) + { + continue; + } + + var lastModified = record.Headers is not null + && record.Headers.TryGetValue("Last-Modified", out var lastModifiedHeader) + && DateTimeOffset.TryParse(lastModifiedHeader, out var parsedLastModified) + ? parsedLastModified.ToUniversalTime().ToString("O") + : record.LastModified?.ToUniversalTime().ToString("O"); + + records.Add(new + { + record.Uri, + record.Status, + record.Sha256, + record.ContentType, + LastModified = lastModified, + Metadata = record.Metadata is null + ? null + : record.Metadata + .OrderBy(static pair => pair.Key, StringComparer.OrdinalIgnoreCase) + .ToDictionary(static pair => pair.Key, static pair => pair.Value, StringComparer.OrdinalIgnoreCase), + record.Etag, + }); + } + + var ordered = records + .OrderBy(static entry => entry.GetType().GetProperty("Uri")?.GetValue(entry)?.ToString(), StringComparer.Ordinal) + .ToArray(); + + return SnapshotSerializer.ToSnapshot(ordered); + } + + private static async Task BuildStateSnapshotAsync(IServiceProvider provider) + { + var stateRepository = provider.GetRequiredService(); + var state = await stateRepository.TryGetAsync(CertCcConnectorPlugin.SourceName, CancellationToken.None); + Assert.NotNull(state); + + var cursor = state!.Cursor ?? new BsonDocument(); + + BsonDocument? summaryDocument = null; + if (cursor.TryGetValue("summary", out var summaryValue) && summaryValue is BsonDocument summaryDoc) + { + summaryDocument = summaryDoc; + } + + var summary = summaryDocument is null + ? null + : new + { + Start = summaryDocument.TryGetValue("start", out var startValue) ? ToIsoString(startValue) : null, + End = summaryDocument.TryGetValue("end", out var endValue) ? ToIsoString(endValue) : null, + }; + + var snapshot = new + { + Summary = summary, + PendingNotes = cursor.TryGetValue("pendingNotes", out var pendingNotesValue) + ? pendingNotesValue.AsBsonArray.Select(static value => value.ToString()).OrderBy(static note => note, StringComparer.OrdinalIgnoreCase).ToArray() + : Array.Empty(), + PendingSummaries = cursor.TryGetValue("pendingSummaries", out var pendingSummariesValue) + ? pendingSummariesValue.AsBsonArray.Select(static value => value.ToString()).OrderBy(static item => item, StringComparer.OrdinalIgnoreCase).ToArray() + : Array.Empty(), + LastRun = cursor.TryGetValue("lastRun", out var lastRunValue) ? ToIsoString(lastRunValue) : null, + state.LastSuccess, + state.LastFailure, + state.FailCount, + state.BackoffUntil, + }; + + return SnapshotSerializer.ToSnapshot(snapshot); + } + + private static async Task BuildAdvisoriesSnapshotAsync(IServiceProvider provider) + { + var advisoryStore = provider.GetRequiredService(); + var advisories = new List(); + await foreach (var advisory in advisoryStore.StreamAsync(CancellationToken.None)) + { + advisories.Add(advisory); + } + + var ordered = advisories + .OrderBy(static advisory => advisory.AdvisoryKey, StringComparer.Ordinal) + .ToArray(); + + return SnapshotSerializer.ToSnapshot(ordered); + } + + private static string BuildRequestsSnapshot(IReadOnlyCollection requests) + { + var ordered = requests + .OrderBy(static request => request.Timestamp) + .Select(static request => new + { + request.Method.Method, + Uri = request.Uri.ToString(), + Headers = new + { + Accept = TryGetHeader(request.Headers, "Accept"), + IfNoneMatch = TryGetHeader(request.Headers, "If-None-Match"), + IfModifiedSince = TryGetHeader(request.Headers, "If-Modified-Since"), + }, + }) + .ToArray(); + + return SnapshotSerializer.ToSnapshot(ordered); + } + + private static void RegisterSummaryResponses(CannedHttpMessageHandler handler) + { + AddJsonResponse(handler, SeptemberSummaryUri, "summary-2025-09.json", "\"certcc-summary-2025-09\"", new DateTimeOffset(2025, 9, 30, 12, 0, 0, TimeSpan.Zero)); + AddJsonResponse(handler, OctoberSummaryUri, "summary-2025-10.json", "\"certcc-summary-2025-10\"", new DateTimeOffset(2025, 10, 31, 12, 0, 0, TimeSpan.Zero)); + var novemberUri = new Uri("https://www.kb.cert.org/vuls/api/2025/11/summary/"); + AddJsonResponse(handler, novemberUri, "summary-2025-11.json", "\"certcc-summary-2025-11\"", new DateTimeOffset(2025, 11, 1, 8, 0, 0, TimeSpan.Zero)); + AddJsonResponse(handler, YearlySummaryUri, "summary-2025.json", "\"certcc-summary-2025\"", new DateTimeOffset(2025, 10, 31, 12, 1, 0, TimeSpan.Zero)); + } + + private static void RegisterSummaryNotModifiedResponses(CannedHttpMessageHandler handler) + { + AddNotModified(handler, SeptemberSummaryUri, "\"certcc-summary-2025-09\""); + AddNotModified(handler, OctoberSummaryUri, "\"certcc-summary-2025-10\""); + var novemberUri = new Uri("https://www.kb.cert.org/vuls/api/2025/11/summary/"); + AddNotModified(handler, novemberUri, "\"certcc-summary-2025-11\""); + AddNotModified(handler, YearlySummaryUri, "\"certcc-summary-2025\""); + } + + private static void RegisterDetailResponses(CannedHttpMessageHandler handler) + { + AddJsonResponse(handler, NoteDetailUri, "vu-294418.json", "\"certcc-note-294418\"", new DateTimeOffset(2025, 10, 9, 16, 52, 0, TimeSpan.Zero)); + AddJsonResponse(handler, VendorsDetailUri, "vu-294418-vendors.json", "\"certcc-vendors-294418\"", new DateTimeOffset(2025, 10, 9, 17, 5, 0, TimeSpan.Zero)); + AddJsonResponse(handler, VulsDetailUri, "vu-294418-vuls.json", "\"certcc-vuls-294418\"", new DateTimeOffset(2025, 10, 9, 17, 10, 0, TimeSpan.Zero)); + AddJsonResponse(handler, VendorStatusesDetailUri, "vendor-statuses-294418.json", "\"certcc-vendor-statuses-294418\"", new DateTimeOffset(2025, 10, 9, 17, 12, 0, TimeSpan.Zero)); + } + + private static void RegisterDetailNotModifiedResponses(CannedHttpMessageHandler handler) + { + AddNotModified(handler, NoteDetailUri, "\"certcc-note-294418\""); + AddNotModified(handler, VendorsDetailUri, "\"certcc-vendors-294418\""); + AddNotModified(handler, VulsDetailUri, "\"certcc-vuls-294418\""); + AddNotModified(handler, VendorStatusesDetailUri, "\"certcc-vendor-statuses-294418\""); + } + + private static void AddJsonResponse(CannedHttpMessageHandler handler, Uri uri, string fixtureName, string etag, DateTimeOffset lastModified) + { + var payload = ReadFixture(fixtureName); + handler.AddResponse(HttpMethod.Get, uri, _ => + { + var response = new HttpResponseMessage(HttpStatusCode.OK) + { + Content = new StringContent(payload, Encoding.UTF8, "application/json"), + }; + response.Headers.ETag = new EntityTagHeaderValue(etag); + response.Headers.TryAddWithoutValidation("Last-Modified", lastModified.ToString("R")); + response.Content.Headers.LastModified = lastModified; + return response; + }); + } + + private static void AddNotModified(CannedHttpMessageHandler handler, Uri uri, string etag) + { + handler.AddResponse(HttpMethod.Get, uri, _ => + { + var response = new HttpResponseMessage(HttpStatusCode.NotModified); + response.Headers.ETag = new EntityTagHeaderValue(etag); + return response; + }); + } + + private static string ReadFixture(string filename) + { + var baseDir = AppContext.BaseDirectory; + var primary = Path.Combine(baseDir, "Fixtures", filename); + if (File.Exists(primary)) + { + return File.ReadAllText(primary); + } + + var fallback = Path.Combine(baseDir, "Source", "CertCc", "Fixtures", filename); + if (File.Exists(fallback)) + { + return File.ReadAllText(fallback); + } + + throw new FileNotFoundException($"Missing CERT/CC fixture '{filename}'."); + } + + private static string? TryGetHeader(IReadOnlyDictionary headers, string key) + => headers.TryGetValue(key, out var value) ? value : null; + + private static string? ToIsoString(BsonValue value) + { + return value.BsonType switch + { + BsonType.DateTime => value.ToUniversalTime().ToString("O"), + BsonType.String when DateTimeOffset.TryParse(value.AsString, out var parsed) => parsed.ToUniversalTime().ToString("O"), + _ => null, + }; + } + + private static void WriteOrAssertSnapshot(string snapshot, string filename) + { + var normalizedSnapshot = Normalize(snapshot); + if (ShouldUpdateFixtures() || !FixtureExists(filename)) + { + var path = GetWritablePath(filename); + Directory.CreateDirectory(Path.GetDirectoryName(path)!); + File.WriteAllText(path, normalizedSnapshot); + return; + } + + var expected = ReadFixture(filename); + var normalizedExpected = Normalize(expected); + if (!string.Equals(normalizedExpected, normalizedSnapshot, StringComparison.Ordinal)) + { + var actualPath = Path.Combine(Path.GetDirectoryName(GetWritablePath(filename))!, Path.GetFileNameWithoutExtension(filename) + ".actual.json"); + Directory.CreateDirectory(Path.GetDirectoryName(actualPath)!); + File.WriteAllText(actualPath, normalizedSnapshot); + } + + Assert.Equal(normalizedExpected, normalizedSnapshot); + } + + private static string GetWritablePath(string filename) + { + var baseDir = AppContext.BaseDirectory; + return Path.Combine(baseDir, "Source", "CertCc", "Fixtures", filename); + } + + private static string Normalize(string value) + => value.Replace("\r\n", "\n", StringComparison.Ordinal).TrimEnd(); + + private static bool ShouldUpdateFixtures() + { + var flag = Environment.GetEnvironmentVariable("UPDATE_CERTCC_FIXTURES"); + return string.Equals(flag, "1", StringComparison.Ordinal) || string.Equals(flag, "true", StringComparison.OrdinalIgnoreCase); + } + + private static bool FixtureExists(string filename) + { + var baseDir = AppContext.BaseDirectory; + var primary = Path.Combine(baseDir, "Source", "CertCc", "Fixtures", filename); + if (File.Exists(primary)) + { + return true; + } + + var fallback = Path.Combine(baseDir, "Fixtures", filename); + return File.Exists(fallback); + } + + public Task InitializeAsync() => Task.CompletedTask; + + public async Task DisposeAsync() + { + if (_harness is not null) + { + await _harness.DisposeAsync(); + } + } +} diff --git a/src/StellaOps.Feedser.Source.CertCc.Tests/CertCc/CertCcConnectorTests.cs b/src/StellaOps.Feedser.Source.CertCc.Tests/CertCc/CertCcConnectorTests.cs new file mode 100644 index 00000000..66bbc5b4 --- /dev/null +++ b/src/StellaOps.Feedser.Source.CertCc.Tests/CertCc/CertCcConnectorTests.cs @@ -0,0 +1,404 @@ +using System; +using System.IO; +using System.Linq; +using System.Net; +using System.Net.Http; +using System.Net.Http.Headers; +using System.Text; +using System.Threading; +using System.Threading.Tasks; +using FluentAssertions; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Http; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Logging.Abstractions; +using Microsoft.Extensions.Time.Testing; +using MongoDB.Bson; +using MongoDB.Driver; +using StellaOps.Feedser.Source.CertCc; +using StellaOps.Feedser.Source.CertCc.Configuration; +using StellaOps.Feedser.Source.Common; +using StellaOps.Feedser.Source.Common.Cursors; +using StellaOps.Feedser.Source.Common.Http; +using StellaOps.Feedser.Source.Common.Testing; +using StellaOps.Feedser.Storage.Mongo; +using StellaOps.Feedser.Storage.Mongo.Documents; +using StellaOps.Feedser.Storage.Mongo.Advisories; +using StellaOps.Feedser.Testing; +using Xunit; + +namespace StellaOps.Feedser.Source.CertCc.Tests.CertCc; + +[Collection("mongo-fixture")] +public sealed class CertCcConnectorTests : IAsyncLifetime +{ + private static readonly Uri MonthlySummaryUri = new("https://www.kb.cert.org/vuls/api/2025/10/summary/"); + private static readonly Uri YearlySummaryUri = new("https://www.kb.cert.org/vuls/api/2025/summary/"); + private static readonly Uri NoteDetailUri = new("https://www.kb.cert.org/vuls/api/294418/"); + private static readonly Uri VendorsUri = new("https://www.kb.cert.org/vuls/api/294418/vendors/"); + private static readonly Uri VulsUri = new("https://www.kb.cert.org/vuls/api/294418/vuls/"); + private static readonly Uri VendorStatusesUri = new("https://www.kb.cert.org/vuls/api/294418/vendors/vuls/"); + + private readonly MongoIntegrationFixture _fixture; + private readonly FakeTimeProvider _timeProvider; + private readonly CannedHttpMessageHandler _handler; + + public CertCcConnectorTests(MongoIntegrationFixture fixture) + { + _fixture = fixture; + _timeProvider = new FakeTimeProvider(new DateTimeOffset(2025, 10, 11, 9, 30, 0, TimeSpan.Zero)); + _handler = new CannedHttpMessageHandler(); + } + + [Fact] + public async Task FetchParseMap_ProducesCanonicalAdvisory() + { + await using var provider = await BuildServiceProviderAsync(); + SeedSummaryResponses(); + SeedDetailResponses(); + + var connector = provider.GetRequiredService(); + await connector.FetchAsync(provider, CancellationToken.None); + await connector.ParseAsync(provider, CancellationToken.None); + await connector.MapAsync(provider, CancellationToken.None); + + var advisoryStore = provider.GetRequiredService(); + var advisories = await advisoryStore.GetRecentAsync(10, CancellationToken.None); + advisories.Should().NotBeNull(); + advisories.Should().HaveCountGreaterThan(0); + + var advisory = advisories.FirstOrDefault(a => a.AdvisoryKey == "certcc/vu-294418"); + advisory.Should().NotBeNull(); + advisory!.Title.Should().ContainEquivalentOf("DrayOS"); + advisory.Summary.Should().NotBeNullOrWhiteSpace(); + advisory.Aliases.Should().Contain("VU#294418"); + advisory.Aliases.Should().Contain("CVE-2025-10547"); + advisory.AffectedPackages.Should().NotBeNull(); + advisory.AffectedPackages.Should().HaveCountGreaterThan(0); + advisory.AffectedPackages![0].NormalizedVersions.Should().NotBeNull(); + + var stateRepository = provider.GetRequiredService(); + var state = await stateRepository.TryGetAsync(CertCcConnectorPlugin.SourceName, CancellationToken.None); + state.Should().NotBeNull(); + var pendingDocuments = state!.Cursor.TryGetValue("pendingDocuments", out var pendingDocsValue) + ? pendingDocsValue!.AsBsonArray.Count + : 0; + pendingDocuments.Should().Be(0); + var pendingMappings = state.Cursor.TryGetValue("pendingMappings", out var pendingMappingsValue) + ? pendingMappingsValue!.AsBsonArray.Count + : 0; + pendingMappings.Should().Be(0); + } + + [Fact] + public async Task Fetch_PersistsSummaryAndDetailDocuments() + { + await using var provider = await BuildServiceProviderAsync(); + SeedSummaryResponses(); + SeedDetailResponses(); + + var connector = provider.GetRequiredService(); + await connector.FetchAsync(provider, CancellationToken.None); + + var documentStore = provider.GetRequiredService(); + + var summaryDocument = await documentStore.FindBySourceAndUriAsync(CertCcConnectorPlugin.SourceName, MonthlySummaryUri.ToString(), CancellationToken.None); + summaryDocument.Should().NotBeNull(); + summaryDocument!.Status.Should().Be(DocumentStatuses.PendingParse); + + var noteDocument = await documentStore.FindBySourceAndUriAsync(CertCcConnectorPlugin.SourceName, NoteDetailUri.ToString(), CancellationToken.None); + noteDocument.Should().NotBeNull(); + noteDocument!.Status.Should().Be(DocumentStatuses.PendingParse); + noteDocument.Metadata.Should().NotBeNull(); + noteDocument.Metadata!.Should().ContainKey("certcc.endpoint").WhoseValue.Should().Be("note"); + noteDocument.Metadata.Should().ContainKey("certcc.noteId").WhoseValue.Should().Be("294418"); + noteDocument.Metadata.Should().ContainKey("certcc.vuid").WhoseValue.Should().Be("VU#294418"); + + var vendorsDocument = await documentStore.FindBySourceAndUriAsync(CertCcConnectorPlugin.SourceName, VendorsUri.ToString(), CancellationToken.None); + vendorsDocument.Should().NotBeNull(); + vendorsDocument!.Status.Should().Be(DocumentStatuses.PendingParse); + vendorsDocument.Metadata.Should().NotBeNull(); + vendorsDocument.Metadata!.Should().ContainKey("certcc.endpoint").WhoseValue.Should().Be("vendors"); + + var vulsDocument = await documentStore.FindBySourceAndUriAsync(CertCcConnectorPlugin.SourceName, VulsUri.ToString(), CancellationToken.None); + vulsDocument.Should().NotBeNull(); + vulsDocument!.Status.Should().Be(DocumentStatuses.PendingParse); + vulsDocument.Metadata.Should().NotBeNull(); + vulsDocument.Metadata!.Should().ContainKey("certcc.endpoint").WhoseValue.Should().Be("vuls"); + + var vendorStatusesDocument = await documentStore.FindBySourceAndUriAsync(CertCcConnectorPlugin.SourceName, VendorStatusesUri.ToString(), CancellationToken.None); + vendorStatusesDocument.Should().NotBeNull(); + vendorStatusesDocument!.Status.Should().Be(DocumentStatuses.PendingParse); + vendorStatusesDocument.Metadata.Should().NotBeNull(); + vendorStatusesDocument.Metadata!.Should().ContainKey("certcc.endpoint").WhoseValue.Should().Be("vendors-vuls"); + + var stateRepository = provider.GetRequiredService(); + var state = await stateRepository.TryGetAsync(CertCcConnectorPlugin.SourceName, CancellationToken.None); + state.Should().NotBeNull(); + state!.Cursor.Should().NotBeNull(); + var pendingNotesCount = state.Cursor.TryGetValue("pendingNotes", out var pendingNotesValue) + ? pendingNotesValue!.AsBsonArray.Count + : 0; + pendingNotesCount.Should().Be(0); + var pendingSummariesCount = state.Cursor.TryGetValue("pendingSummaries", out var pendingSummariesValue) + ? pendingSummariesValue!.AsBsonArray.Count + : 0; + pendingSummariesCount.Should().Be(0); + + var pendingDocumentsCount = state.Cursor.TryGetValue("pendingDocuments", out var pendingDocumentsValue) + ? pendingDocumentsValue!.AsBsonArray.Count + : 0; + pendingDocumentsCount.Should().Be(4); + + var pendingMappingsCount = state.Cursor.TryGetValue("pendingMappings", out var pendingMappingsValue) + ? pendingMappingsValue!.AsBsonArray.Count + : 0; + pendingMappingsCount.Should().Be(0); + + _handler.Requests.Should().Contain(request => request.Uri == NoteDetailUri); + } + + [Fact] + public async Task Fetch_ReusesConditionalRequestsOnSubsequentRun() + { + await using var provider = await BuildServiceProviderAsync(); + SeedSummaryResponses(summaryEtag: "\"summary-oct\"", yearlyEtag: "\"summary-year\""); + SeedDetailResponses(detailEtag: "\"note-etag\"", vendorsEtag: "\"vendors-etag\"", vulsEtag: "\"vuls-etag\"", vendorStatusesEtag: "\"vendor-statuses-etag\""); + + var connector = provider.GetRequiredService(); + await connector.FetchAsync(provider, CancellationToken.None); + + _handler.Clear(); + SeedSummaryNotModifiedResponses("\"summary-oct\"", "\"summary-year\""); + SeedDetailNotModifiedResponses("\"note-etag\"", "\"vendors-etag\"", "\"vuls-etag\"", "\"vendor-statuses-etag\""); + _timeProvider.Advance(TimeSpan.FromMinutes(15)); + + await connector.FetchAsync(provider, CancellationToken.None); + + var requests = _handler.Requests.ToArray(); + requests.Should().OnlyContain(r => + r.Uri == MonthlySummaryUri + || r.Uri == YearlySummaryUri + || r.Uri == NoteDetailUri + || r.Uri == VendorsUri + || r.Uri == VulsUri + || r.Uri == VendorStatusesUri); + + var stateRepository = provider.GetRequiredService(); + var state = await stateRepository.TryGetAsync(CertCcConnectorPlugin.SourceName, CancellationToken.None); + state.Should().NotBeNull(); + var pendingNotesCount = state!.Cursor.TryGetValue("pendingNotes", out var pendingNotesValue) + ? pendingNotesValue!.AsBsonArray.Count + : 0; + pendingNotesCount.Should().Be(0); + var pendingSummaries = state.Cursor.TryGetValue("pendingSummaries", out var pendingSummariesValue) + ? pendingSummariesValue!.AsBsonArray.Count + : 0; + pendingSummaries.Should().Be(0); + + var pendingDocuments = state.Cursor.TryGetValue("pendingDocuments", out var pendingDocumentsValue) + ? pendingDocumentsValue!.AsBsonArray.Count + : 0; + pendingDocuments.Should().BeGreaterThan(0); + } + + [Fact] + public async Task Fetch_DetailFailureRecordsBackoffAndKeepsPendingNote() + { + await using var provider = await BuildServiceProviderAsync(); + SeedSummaryResponses(); + SeedDetailResponses(vendorsStatus: HttpStatusCode.InternalServerError); + + var connector = provider.GetRequiredService(); + var failure = await Assert.ThrowsAnyAsync(() => connector.FetchAsync(provider, CancellationToken.None)); + Assert.True(failure is HttpRequestException || failure is InvalidOperationException); + + var stateRepository = provider.GetRequiredService(); + var state = await stateRepository.TryGetAsync(CertCcConnectorPlugin.SourceName, CancellationToken.None); + state.Should().NotBeNull(); + state!.FailCount.Should().BeGreaterThan(0); + state.BackoffUntil.Should().NotBeNull(); + state.BackoffUntil.Should().BeAfter(_timeProvider.GetUtcNow()); + state.Cursor.TryGetValue("pendingNotes", out var pendingNotesValue).Should().BeTrue(); + pendingNotesValue!.AsBsonArray.Should().Contain(value => value.AsString == "294418"); + var pendingSummaries = state.Cursor.TryGetValue("pendingSummaries", out var pendingSummariesValue) + ? pendingSummariesValue!.AsBsonArray.Count + : 0; + pendingSummaries.Should().Be(0); + } + + public Task InitializeAsync() => Task.CompletedTask; + + public async Task DisposeAsync() + { + await _fixture.Client.DropDatabaseAsync(_fixture.Database.DatabaseNamespace.DatabaseName); + } + + [Fact] + public async Task ParseAndMap_SkipWhenDetailMappingDisabled() + { + await using var provider = await BuildServiceProviderAsync(enableDetailMapping: false); + SeedSummaryResponses(); + SeedDetailResponses(); + + var connector = provider.GetRequiredService(); + await connector.FetchAsync(provider, CancellationToken.None); + await connector.ParseAsync(provider, CancellationToken.None); + await connector.MapAsync(provider, CancellationToken.None); + + var advisoryStore = provider.GetRequiredService(); + var advisories = await advisoryStore.GetRecentAsync(10, CancellationToken.None); + advisories.Should().BeNullOrEmpty(); + + var documentStore = provider.GetRequiredService(); + var noteDocument = await documentStore.FindBySourceAndUriAsync(CertCcConnectorPlugin.SourceName, NoteDetailUri.ToString(), CancellationToken.None); + noteDocument.Should().NotBeNull(); + noteDocument!.Status.Should().Be(DocumentStatuses.PendingParse); + + var stateRepository = provider.GetRequiredService(); + var state = await stateRepository.TryGetAsync(CertCcConnectorPlugin.SourceName, CancellationToken.None); + state.Should().NotBeNull(); + var pendingDocuments = state!.Cursor.TryGetValue("pendingDocuments", out var pendingDocsValue) + ? pendingDocsValue!.AsBsonArray.Count + : 0; + pendingDocuments.Should().BeGreaterThan(0); + var pendingMappings = state.Cursor.TryGetValue("pendingMappings", out var pendingMappingsValue) + ? pendingMappingsValue!.AsBsonArray.Count + : 0; + pendingMappings.Should().Be(0); + } + + private async Task BuildServiceProviderAsync(bool enableDetailMapping = true) + { + await _fixture.Client.DropDatabaseAsync(_fixture.Database.DatabaseNamespace.DatabaseName); + _handler.Clear(); + + var services = new ServiceCollection(); + services.AddLogging(builder => builder.AddProvider(NullLoggerProvider.Instance)); + services.AddSingleton(_timeProvider); + services.AddSingleton(_handler); + + services.AddMongoStorage(options => + { + options.ConnectionString = _fixture.Runner.ConnectionString; + options.DatabaseName = _fixture.Database.DatabaseNamespace.DatabaseName; + options.CommandTimeout = TimeSpan.FromSeconds(5); + }); + + services.AddSourceCommon(); + services.AddCertCcConnector(options => + { + options.BaseApiUri = new Uri("https://www.kb.cert.org/vuls/api/"); + options.SummaryWindow = new TimeWindowCursorOptions + { + WindowSize = TimeSpan.FromDays(1), + Overlap = TimeSpan.Zero, + InitialBackfill = TimeSpan.FromDays(1), + MinimumWindowSize = TimeSpan.FromHours(6), + }; + options.MaxMonthlySummaries = 1; + options.MaxNotesPerFetch = 5; + options.DetailRequestDelay = TimeSpan.Zero; + options.EnableDetailMapping = enableDetailMapping; + }); + + services.Configure(CertCcOptions.HttpClientName, builderOptions => + { + builderOptions.HttpMessageHandlerBuilderActions.Add(builder => + { + builder.PrimaryHandler = _handler; + }); + }); + + var provider = services.BuildServiceProvider(); + var bootstrapper = provider.GetRequiredService(); + await bootstrapper.InitializeAsync(CancellationToken.None); + return provider; + } + + private void SeedSummaryResponses(string summaryEtag = "\"summary-oct\"", string yearlyEtag = "\"summary-year\"") + { + AddJsonResponse(MonthlySummaryUri, ReadFixture("summary-2025-10.json"), summaryEtag); + AddJsonResponse(YearlySummaryUri, ReadFixture("summary-2025.json"), yearlyEtag); + } + + private void SeedSummaryNotModifiedResponses(string summaryEtag, string yearlyEtag) + { + AddNotModifiedResponse(MonthlySummaryUri, summaryEtag); + AddNotModifiedResponse(YearlySummaryUri, yearlyEtag); + } + + private void SeedDetailResponses( + string detailEtag = "\"note-etag\"", + string vendorsEtag = "\"vendors-etag\"", + string vulsEtag = "\"vuls-etag\"", + string vendorStatusesEtag = "\"vendor-statuses-etag\"", + HttpStatusCode vendorsStatus = HttpStatusCode.OK) + { + AddJsonResponse(NoteDetailUri, ReadFixture("vu-294418.json"), detailEtag); + + if (vendorsStatus == HttpStatusCode.OK) + { + AddJsonResponse(VendorsUri, ReadFixture("vu-294418-vendors.json"), vendorsEtag); + } + else + { + _handler.AddResponse(VendorsUri, () => + { + var response = new HttpResponseMessage(vendorsStatus) + { + Content = new StringContent("vendors error", Encoding.UTF8, "text/plain"), + }; + response.Headers.ETag = new EntityTagHeaderValue(vendorsEtag); + return response; + }); + } + + AddJsonResponse(VulsUri, ReadFixture("vu-294418-vuls.json"), vulsEtag); + AddJsonResponse(VendorStatusesUri, ReadFixture("vendor-statuses-294418.json"), vendorStatusesEtag); + } + + private void SeedDetailNotModifiedResponses(string detailEtag, string vendorsEtag, string vulsEtag, string vendorStatusesEtag) + { + AddNotModifiedResponse(NoteDetailUri, detailEtag); + AddNotModifiedResponse(VendorsUri, vendorsEtag); + AddNotModifiedResponse(VulsUri, vulsEtag); + AddNotModifiedResponse(VendorStatusesUri, vendorStatusesEtag); + } + + private void AddJsonResponse(Uri uri, string json, string etag) + { + _handler.AddResponse(uri, () => + { + var response = new HttpResponseMessage(HttpStatusCode.OK) + { + Content = new StringContent(json, Encoding.UTF8, "application/json"), + }; + response.Headers.ETag = new EntityTagHeaderValue(etag); + return response; + }); + } + + private void AddNotModifiedResponse(Uri uri, string etag) + { + _handler.AddResponse(uri, () => + { + var response = new HttpResponseMessage(HttpStatusCode.NotModified); + response.Headers.ETag = new EntityTagHeaderValue(etag); + return response; + }); + } + + private static string ReadFixture(string filename) + { + var baseDirectory = AppContext.BaseDirectory; + var candidate = Path.Combine(baseDirectory, "Source", "CertCc", "Fixtures", filename); + if (File.Exists(candidate)) + { + return File.ReadAllText(candidate); + } + + var fallback = Path.Combine(baseDirectory, "Fixtures", filename); + return File.ReadAllText(fallback); + } +} diff --git a/src/StellaOps.Feedser.Source.CertCc.Tests/Fixtures/certcc-documents.snapshot.json b/src/StellaOps.Feedser.Source.CertCc.Tests/Fixtures/certcc-documents.snapshot.json new file mode 100644 index 00000000..87660272 --- /dev/null +++ b/src/StellaOps.Feedser.Source.CertCc.Tests/Fixtures/certcc-documents.snapshot.json @@ -0,0 +1,91 @@ +[ + { + "contentType": "application/json; charset=utf-8", + "etag": "\"certcc-summary-2025-09\"", + "lastModified": "2025-09-30T12:00:00.0000000+00:00", + "metadata": { + "attempts": "1", + "certcc.month": "09", + "certcc.scope": "monthly", + "certcc.year": "2025", + "fetchedAt": "2025-11-01T08:00:00.0000000+00:00" + }, + "sha256": "75e6271ec1a0a3a099d13aa23393d8ddf5fa9638a06748a84538f9b8891ea792", + "status": "pending-parse", + "uri": "https://www.kb.cert.org/vuls/api/2025/09/summary/" + }, + { + "contentType": "application/json; charset=utf-8", + "etag": "\"certcc-summary-2025-10\"", + "lastModified": "2025-10-31T12:00:00.0000000+00:00", + "metadata": { + "attempts": "1", + "certcc.month": "10", + "certcc.scope": "monthly", + "certcc.year": "2025", + "fetchedAt": "2025-11-01T08:00:00.0000000+00:00" + }, + "sha256": "75e6271ec1a0a3a099d13aa23393d8ddf5fa9638a06748a84538f9b8891ea792", + "status": "pending-parse", + "uri": "https://www.kb.cert.org/vuls/api/2025/10/summary/" + }, + { + "contentType": "application/json; charset=utf-8", + "etag": "\"certcc-summary-2025\"", + "lastModified": "2025-10-31T12:01:00.0000000+00:00", + "metadata": { + "attempts": "1", + "certcc.scope": "yearly", + "certcc.year": "2025", + "fetchedAt": "2025-11-01T08:00:00.0000000+00:00" + }, + "sha256": "75e6271ec1a0a3a099d13aa23393d8ddf5fa9638a06748a84538f9b8891ea792", + "status": "pending-parse", + "uri": "https://www.kb.cert.org/vuls/api/2025/summary/" + }, + { + "contentType": "application/json; charset=utf-8", + "etag": "\"certcc-note-294418\"", + "lastModified": "2025-10-09T16:52:00.0000000+00:00", + "metadata": { + "attempts": "1", + "certcc.endpoint": "note", + "certcc.noteId": "294418", + "certcc.vuid": "VU#294418", + "fetchedAt": "2025-11-01T08:00:00.0000000+00:00" + }, + "sha256": "fac27a3caa47fe319a2eb2a88145452daebaf3c8c6b5afe62cf9634b0824c003", + "status": "pending-parse", + "uri": "https://www.kb.cert.org/vuls/api/294418/" + }, + { + "contentType": "application/json; charset=utf-8", + "etag": "\"certcc-vendors-294418\"", + "lastModified": "2025-10-09T17:05:00.0000000+00:00", + "metadata": { + "attempts": "1", + "certcc.endpoint": "vendors", + "certcc.noteId": "294418", + "certcc.vuid": "VU#294418", + "fetchedAt": "2025-11-01T08:00:00.0000000+00:00" + }, + "sha256": "34036a3cdda490adf8ad26d74ac8ae3b85d591579e7dd26b1fd2f78fe5e401b8", + "status": "pending-parse", + "uri": "https://www.kb.cert.org/vuls/api/294418/vendors/" + }, + { + "contentType": "application/json; charset=utf-8", + "etag": "\"certcc-vuls-294418\"", + "lastModified": "2025-10-09T17:10:00.0000000+00:00", + "metadata": { + "attempts": "1", + "certcc.endpoint": "vuls", + "certcc.noteId": "294418", + "certcc.vuid": "VU#294418", + "fetchedAt": "2025-11-01T08:00:00.0000000+00:00" + }, + "sha256": "84c7c17fc37bffdee37cd020b43ec6cadc217a573135ba1c5cc2f0495846030a", + "status": "pending-parse", + "uri": "https://www.kb.cert.org/vuls/api/294418/vuls/" + } +] \ No newline at end of file diff --git a/src/StellaOps.Feedser.Source.CertCc.Tests/Fixtures/certcc-state.snapshot.json b/src/StellaOps.Feedser.Source.CertCc.Tests/Fixtures/certcc-state.snapshot.json new file mode 100644 index 00000000..2fbc20f4 --- /dev/null +++ b/src/StellaOps.Feedser.Source.CertCc.Tests/Fixtures/certcc-state.snapshot.json @@ -0,0 +1,13 @@ +{ + "backoffUntil": null, + "failCount": 0, + "lastFailure": null, + "lastRun": "2025-11-01T08:00:00.0000000Z", + "lastSuccess": "2025-11-01T08:00:00+00:00", + "pendingNotes": [], + "pendingSummaries": [], + "summary": { + "end": "2025-10-17T08:00:00.0000000Z", + "start": "2025-09-17T08:00:00.0000000Z" + } +} \ No newline at end of file diff --git a/src/StellaOps.Feedser.Source.CertCc.Tests/Fixtures/summary-2025-09.json b/src/StellaOps.Feedser.Source.CertCc.Tests/Fixtures/summary-2025-09.json new file mode 100644 index 00000000..af07696f --- /dev/null +++ b/src/StellaOps.Feedser.Source.CertCc.Tests/Fixtures/summary-2025-09.json @@ -0,0 +1,4 @@ +{ + "count": 0, + "notes": [] +} diff --git a/src/StellaOps.Feedser.Source.CertCc.Tests/Fixtures/summary-2025-10.json b/src/StellaOps.Feedser.Source.CertCc.Tests/Fixtures/summary-2025-10.json new file mode 100644 index 00000000..8f6680a4 --- /dev/null +++ b/src/StellaOps.Feedser.Source.CertCc.Tests/Fixtures/summary-2025-10.json @@ -0,0 +1,6 @@ +{ + "count": 1, + "notes": [ + "VU#294418" + ] +} diff --git a/src/StellaOps.Feedser.Source.CertCc.Tests/Fixtures/summary-2025-11.json b/src/StellaOps.Feedser.Source.CertCc.Tests/Fixtures/summary-2025-11.json new file mode 100644 index 00000000..af07696f --- /dev/null +++ b/src/StellaOps.Feedser.Source.CertCc.Tests/Fixtures/summary-2025-11.json @@ -0,0 +1,4 @@ +{ + "count": 0, + "notes": [] +} diff --git a/src/StellaOps.Feedser.Source.CertCc.Tests/Fixtures/summary-2025.json b/src/StellaOps.Feedser.Source.CertCc.Tests/Fixtures/summary-2025.json new file mode 100644 index 00000000..8f6680a4 --- /dev/null +++ b/src/StellaOps.Feedser.Source.CertCc.Tests/Fixtures/summary-2025.json @@ -0,0 +1,6 @@ +{ + "count": 1, + "notes": [ + "VU#294418" + ] +} diff --git a/src/StellaOps.Feedser.Source.CertCc.Tests/Fixtures/vendor-statuses-294418.json b/src/StellaOps.Feedser.Source.CertCc.Tests/Fixtures/vendor-statuses-294418.json new file mode 100644 index 00000000..4fdb02bf --- /dev/null +++ b/src/StellaOps.Feedser.Source.CertCc.Tests/Fixtures/vendor-statuses-294418.json @@ -0,0 +1,11 @@ +[ + { + "vul": "CVE-2025-10547", + "vendor": "DrayTek Corporation", + "status": "Affected", + "date_added": "2025-10-03T11:35:31.202991Z", + "dateupdated": "2025-10-03T11:40:09.944401Z", + "references": null, + "statement": null + } +] diff --git a/src/StellaOps.Feedser.Source.CertCc.Tests/Fixtures/vendors-294418.json b/src/StellaOps.Feedser.Source.CertCc.Tests/Fixtures/vendors-294418.json new file mode 100644 index 00000000..e26d82cd --- /dev/null +++ b/src/StellaOps.Feedser.Source.CertCc.Tests/Fixtures/vendors-294418.json @@ -0,0 +1,12 @@ +[ + { + "note": "294418", + "contact_date": "2025-09-15T19:03:33.664345Z", + "vendor": "DrayTek Corporation", + "references": "", + "statement": "The issue is confirmed, and here is the patch list\r\n\r\nV3912/V3910/V2962/V1000B\t4.4.3.6/4.4.5.1\r\nV2927/V2865/V2866\t4.5.1\r\nV2765/V2766/V2763/V2135\t4.5.1\r\nV2915\t4.4.6.1\r\nV2862/V2926\t3.9.9.12\r\nV2952/3220\t3.9.8.8\r\nV2860/V2925\t3.9.8.6\r\nV2133/V2762/V2832\t3.9.9.4\r\nV2620/LTE200\t3.9.9.5", + "dateupdated": "2025-10-03T11:35:31.190661Z", + "statement_date": "2025-09-16T02:27:51.346335Z", + "addendum": null + } +] diff --git a/src/StellaOps.Feedser.Source.CertCc.Tests/Fixtures/vu-257161.json b/src/StellaOps.Feedser.Source.CertCc.Tests/Fixtures/vu-257161.json new file mode 100644 index 00000000..c72f66e7 --- /dev/null +++ b/src/StellaOps.Feedser.Source.CertCc.Tests/Fixtures/vu-257161.json @@ -0,0 +1,87 @@ +{ + "vuid": "VU#257161", + "idnumber": "257161", + "name": "Treck IP stacks contain multiple vulnerabilities", + "keywords": null, + "overview": "### Overview\r\nTreck IP stack implementations for embedded systems are affected by multiple vulnerabilities. This set of vulnerabilities was researched and reported by JSOF, who calls them [Ripple20](https://www.jsof-tech.com/ripple20/).\r\n\r\n### Description\r\nTreck IP network stack software is designed for and used in a variety of embedded systems. The software can be licensed and integrated in various ways, including compiled from source, licensed for modification and reuse and finally as a dynamic or static linked library. Treck IP software contains multiple vulnerabilities, most of which are caused by [memory management bugs](https://wiki.sei.cmu.edu/confluence/pages/viewpage.action?pageId=87152142). For more details on the vulnerabilities introduced by these bugs, see Treck's [ Vulnerability Response Information](https://treck.com/vulnerability-response-information/) and JSOF's [Ripple20 advisory](https://www.jsof-tech.com/ripple20/).\r\n\r\nHistorically-related KASAGO TCP/IP middleware from Zuken Elmic (formerly Elmic Systems) is also affected by some of these vulnerabilities. \r\n\r\nThese vulnerabilities likely affect industrial control systems and medical devices. Please see ICS-CERT Advisory [ICSA-20-168-01](https://www.us-cert.gov/ics/advisories/icsa-20-168-01) for more information.\r\n\r\n### Impact ###\r\nThe impact of these vulnerabilities will vary due to the combination of build and runtime options used while developing different embedded systems. This diversity of implementations and the lack of supply chain visibility has exasperated the problem of accurately assessing the impact of these vulnerabilities. In summary, a remote, unauthenticated attacker may be able to use specially-crafted network packets to cause a denial of service, disclose information, or execute arbitrary code.\r\n\r\n### Solution\r\n#### Apply updates\r\nUpdate to the latest stable version of Treck IP stack software (6.0.1.67 or later). Please contact Treck at . Downstream users of embedded systems that incorporate Treck IP stacks should contact their embedded system vendor.\r\n\r\n#### Block anomalous IP traffic\r\nConsider blocking network attacks via deep packet inspection. In some cases, modern switches, routers, and firewalls will drop malformed packets with no additional configuration. It is recommended that such security features are not disabled. Below is a list of possible mitigations that can be applied as appropriate to your network environment.\r\n\r\n* Normalize or reject IP fragmented packets (IP Fragments) if not supported in your environment \r\n* Disable or block IP tunneling, both IPv6-in-IPv4 or IP-in-IP tunneling if not required\r\n* Block IP source routing and any IPv6 deprecated features like routing headers (see also [VU#267289](https://www.kb.cert.org/vuls/id/267289))\r\n* Enforce TCP inspection and reject malformed TCP packets \r\n* Block unused ICMP control messages such MTU Update and Address Mask updates\r\n* Normalize DNS through a secure recursive server or application layer firewall\r\n* Ensure that you are using reliable OSI layer 2 equipment (Ethernet)\r\n* Provide DHCP/DHCPv6 security with feature like DHCP snooping\r\n* Disable or block IPv6 multicast if not used in switching infrastructure\r\n\r\nFurther recommendations are available [here](https://github.com/CERTCC/PoC-Exploits/blob/master/vu-257161/recommendations.md).\r\n\r\n#### Detect anomalous IP traffic\r\nSuricata IDS has built-in decoder-event rules that can be customized to detect attempts to exploit these vulnerabilities. See the rule below for an example. A larger set of selected [vu-257161.rules](https://github.com/CERTCC/PoC-Exploits/blob/master/vu-257161/vu-257161.rules) are available from the CERT/CC Github repository.\r\n\r\n`#IP-in-IP tunnel with fragments` \r\n`alert ip any any -> any any (msg:\"VU#257161:CVE-2020-11896, CVE-2020-11900 Fragments inside IP-in-IP tunnel https://kb.cert.org/vuls/id/257161\"; ip_proto:4; fragbits:M; sid:1367257161; rev:1;)`\r\n\r\n### Acknowledgements\r\nMoshe Kol and Shlomi Oberman of JSOF https://jsof-tech.com researched and reported these vulnerabilities. Treck worked closely with us and other stakeholders to coordinate the disclosure of these vulnerabilities.\r\n\r\nThis document was written by Vijay Sarvepalli.", + "clean_desc": null, + "impact": null, + "resolution": null, + "workarounds": null, + "sysaffected": null, + "thanks": null, + "author": null, + "public": [ + "https://www.jsof-tech.com/ripple20/", + "https://treck.com/vulnerability-response-information/", + "https://www.us-cert.gov/ics/advisories/icsa-20-168-01", + "https://jvn.jp/vu/JVNVU94736763/index.html" + ], + "cveids": [ + "CVE-2020-11902", + "CVE-2020-11913", + "CVE-2020-11898", + "CVE-2020-11907", + "CVE-2020-11901", + "CVE-2020-11903", + "CVE-2020-11904", + "CVE-2020-11906", + "CVE-2020-11910", + "CVE-2020-11911", + "CVE-2020-11912", + "CVE-2020-11914", + "CVE-2020-11899", + "CVE-2020-11896", + "CVE-2020-11897", + "CVE-2020-11905", + "CVE-2020-11908", + "CVE-2020-11900", + "CVE-2020-11909", + "CVE-2020-0597", + "CVE-2020-0595", + "CVE-2020-8674", + "CVE-2020-0594" + ], + "certadvisory": null, + "uscerttechnicalalert": null, + "datecreated": "2020-06-16T17:13:53.220714Z", + "publicdate": "2020-06-16T00:00:00Z", + "datefirstpublished": "2020-06-16T17:13:53.238540Z", + "dateupdated": "2022-09-20T01:54:35.485507Z", + "revision": 48, + "vrda_d1_directreport": null, + "vrda_d1_population": null, + "vrda_d1_impact": null, + "cam_widelyknown": null, + "cam_exploitation": null, + "cam_internetinfrastructure": null, + "cam_population": null, + "cam_impact": null, + "cam_easeofexploitation": null, + "cam_attackeraccessrequired": null, + "cam_scorecurrent": null, + "cam_scorecurrentwidelyknown": null, + "cam_scorecurrentwidelyknownexploited": null, + "ipprotocol": null, + "cvss_accessvector": null, + "cvss_accesscomplexity": null, + "cvss_authentication": null, + "cvss_confidentialityimpact": null, + "cvss_integrityimpact": null, + "cvss_availabilityimpact": null, + "cvss_exploitablity": null, + "cvss_remediationlevel": null, + "cvss_reportconfidence": null, + "cvss_collateraldamagepotential": null, + "cvss_targetdistribution": null, + "cvss_securityrequirementscr": null, + "cvss_securityrequirementsir": null, + "cvss_securityrequirementsar": null, + "cvss_basescore": null, + "cvss_basevector": null, + "cvss_temporalscore": null, + "cvss_environmentalscore": null, + "cvss_environmentalvector": null, + "metric": null, + "vulnote": 7 +} diff --git a/src/StellaOps.Feedser.Source.CertCc.Tests/Fixtures/vu-294418-vendors.json b/src/StellaOps.Feedser.Source.CertCc.Tests/Fixtures/vu-294418-vendors.json new file mode 100644 index 00000000..e26d82cd --- /dev/null +++ b/src/StellaOps.Feedser.Source.CertCc.Tests/Fixtures/vu-294418-vendors.json @@ -0,0 +1,12 @@ +[ + { + "note": "294418", + "contact_date": "2025-09-15T19:03:33.664345Z", + "vendor": "DrayTek Corporation", + "references": "", + "statement": "The issue is confirmed, and here is the patch list\r\n\r\nV3912/V3910/V2962/V1000B\t4.4.3.6/4.4.5.1\r\nV2927/V2865/V2866\t4.5.1\r\nV2765/V2766/V2763/V2135\t4.5.1\r\nV2915\t4.4.6.1\r\nV2862/V2926\t3.9.9.12\r\nV2952/3220\t3.9.8.8\r\nV2860/V2925\t3.9.8.6\r\nV2133/V2762/V2832\t3.9.9.4\r\nV2620/LTE200\t3.9.9.5", + "dateupdated": "2025-10-03T11:35:31.190661Z", + "statement_date": "2025-09-16T02:27:51.346335Z", + "addendum": null + } +] diff --git a/src/StellaOps.Feedser.Source.CertCc.Tests/Fixtures/vu-294418-vuls.json b/src/StellaOps.Feedser.Source.CertCc.Tests/Fixtures/vu-294418-vuls.json new file mode 100644 index 00000000..ec166e0b --- /dev/null +++ b/src/StellaOps.Feedser.Source.CertCc.Tests/Fixtures/vu-294418-vuls.json @@ -0,0 +1,11 @@ +[ + { + "note": "294418", + "cve": "2025-10547", + "description": "An uninitialized variable in the HTTP CGI request arguments processing component of Vigor Routers running DrayOS may allow an attacker the ability to perform RCE on the appliance through memory corruption.", + "uid": "CVE-2025-10547", + "case_increment": 1, + "date_added": "2025-10-03T11:35:31.177872Z", + "dateupdated": "2025-10-03T11:40:09.915649Z" + } +] diff --git a/src/StellaOps.Feedser.Source.CertCc.Tests/Fixtures/vu-294418.json b/src/StellaOps.Feedser.Source.CertCc.Tests/Fixtures/vu-294418.json new file mode 100644 index 00000000..72493215 --- /dev/null +++ b/src/StellaOps.Feedser.Source.CertCc.Tests/Fixtures/vu-294418.json @@ -0,0 +1,63 @@ +{ + "vuid": "VU#294418", + "idnumber": "294418", + "name": "Vigor routers running DrayOS are vulnerable to RCE via EasyVPN and LAN web administration interface", + "keywords": null, + "overview": "### Overview\r\nA remote code execution (RCE) vulnerability, tracked as CVE-2025-10547, was discovered through the EasyVPN and LAN web administration interface of Vigor routers by Draytek. A script in the LAN web administration interface uses an unitialized variable, allowing an attacker to send specially crafted HTTP requests that cause memory corruption and potentially allow arbitrary code execution.\r\n\t\r\n### Description\r\nVigor routers are business-grade routers, designed for small to medium-sized businesses, made by Draytek. These routers provide routing, firewall, VPN, content-filtering, bandwidth management, LAN (local area network), and multi-WAN (wide area network) features. Draytek utilizes a proprietary firmware, DrayOS, on the Vigor router line. DrayOS features the EasyVPN and LAN Web Administrator tool s to facilitate LAN and VPN setup. According to the DrayTek [website](https://www.draytek.com/support/knowledge-base/12023), \"with EasyVPN, users no longer need to generate WireGuard keys, import OpenVPN configuration files, or upload certificates. Instead, VPN can be successfully established by simply entering the username and password or getting the OTP code by email.\" \r\n\r\nThe LAN Web Administrator provides a browser-based user interface for router management. When a user interacts with the LAN Web Administration interface, the user interface elements trigger actions that generate HTTP requests to interact with the local server. This process contains an uninitialized variable. Due to the uninitialized variable, an unauthenticated attacker could perform memory corruption on the router via specially crafted HTTP requests to hijack execution or inject malicious payloads. If EasyVPN is enabled, the flaw could be remotely exploited through the VPN interface.\r\n\r\n### Impact\r\nA remote, unathenticated attacker can exploit this vulnerability through accessing the LAN interface\u2014or potentially the WAN interface\u2014if EasyVPN is enabled or remote administration over the internet is activated. If a remote, unauthenticated attacker leverages this vulnerability, they can execute arbitrary code on the router (RCE) and gain full control of the device. A successful attack could result in a attacker gaining root access to a Vigor router to then install backdoors, reconfigure network settings, or block traffic. An attacker may also pivot for lateral movement via intercepting internal communications and bypassing VPNs. \r\n\r\n### Solution\r\nThe DrayTek Security team has developed a series of patches to remediate the vulnerability, and all users of Vigor routers should upgrade to the latest version ASAP. The patches can be found on the [resources](https://www.draytek.com/support/resources?type=version) page of the DrayTek webpage, and the security advisory can be found within the [about](https://www.draytek.com/about/security-advisory/use-of-uninitialized-variable-vulnerabilities/) section of the DrayTek webpage. Consult either the CVE [listing](https://nvd.nist.gov/vuln/detail/CVE-2025-10547) or the [advisory page](https://www.draytek.com/about/security-advisory/use-of-uninitialized-variable-vulnerabilities/) for a full list of affected products. \r\n\r\n### Acknowledgements\r\nThanks to the reporter, Pierre-Yves MAES of ChapsVision (pymaes@chapsvision.com). This document was written by Ayushi Kriplani.", + "clean_desc": null, + "impact": null, + "resolution": null, + "workarounds": null, + "sysaffected": null, + "thanks": null, + "author": null, + "public": [ + "https://www.draytek.com/about/security-advisory/use-of-uninitialized-variable-vulnerabilities/", + "https://www.draytek.com/support/resources?type=version" + ], + "cveids": [ + "CVE-2025-10547" + ], + "certadvisory": null, + "uscerttechnicalalert": null, + "datecreated": "2025-10-03T11:35:31.224065Z", + "publicdate": "2025-10-03T11:35:31.026053Z", + "datefirstpublished": "2025-10-03T11:35:31.247121Z", + "dateupdated": "2025-10-03T11:40:09.876722Z", + "revision": 2, + "vrda_d1_directreport": null, + "vrda_d1_population": null, + "vrda_d1_impact": null, + "cam_widelyknown": null, + "cam_exploitation": null, + "cam_internetinfrastructure": null, + "cam_population": null, + "cam_impact": null, + "cam_easeofexploitation": null, + "cam_attackeraccessrequired": null, + "cam_scorecurrent": null, + "cam_scorecurrentwidelyknown": null, + "cam_scorecurrentwidelyknownexploited": null, + "ipprotocol": null, + "cvss_accessvector": null, + "cvss_accesscomplexity": null, + "cvss_authentication": null, + "cvss_confidentialityimpact": null, + "cvss_integrityimpact": null, + "cvss_availabilityimpact": null, + "cvss_exploitablity": null, + "cvss_remediationlevel": null, + "cvss_reportconfidence": null, + "cvss_collateraldamagepotential": null, + "cvss_targetdistribution": null, + "cvss_securityrequirementscr": null, + "cvss_securityrequirementsir": null, + "cvss_securityrequirementsar": null, + "cvss_basescore": null, + "cvss_basevector": null, + "cvss_temporalscore": null, + "cvss_environmentalscore": null, + "cvss_environmentalvector": null, + "metric": null, + "vulnote": 142 +} diff --git a/src/StellaOps.Feedser.Source.CertCc.Tests/Fixtures/vulnerabilities-294418.json b/src/StellaOps.Feedser.Source.CertCc.Tests/Fixtures/vulnerabilities-294418.json new file mode 100644 index 00000000..ec166e0b --- /dev/null +++ b/src/StellaOps.Feedser.Source.CertCc.Tests/Fixtures/vulnerabilities-294418.json @@ -0,0 +1,11 @@ +[ + { + "note": "294418", + "cve": "2025-10547", + "description": "An uninitialized variable in the HTTP CGI request arguments processing component of Vigor Routers running DrayOS may allow an attacker the ability to perform RCE on the appliance through memory corruption.", + "uid": "CVE-2025-10547", + "case_increment": 1, + "date_added": "2025-10-03T11:35:31.177872Z", + "dateupdated": "2025-10-03T11:40:09.915649Z" + } +] diff --git a/src/StellaOps.Feedser.Source.CertCc.Tests/Internal/CertCcMapperTests.cs b/src/StellaOps.Feedser.Source.CertCc.Tests/Internal/CertCcMapperTests.cs new file mode 100644 index 00000000..c9304737 --- /dev/null +++ b/src/StellaOps.Feedser.Source.CertCc.Tests/Internal/CertCcMapperTests.cs @@ -0,0 +1,118 @@ +using System; +using System.Globalization; +using MongoDB.Bson; +using StellaOps.Feedser.Models; +using StellaOps.Feedser.Source.CertCc.Internal; +using StellaOps.Feedser.Storage.Mongo.Documents; +using StellaOps.Feedser.Storage.Mongo.Dtos; +using Xunit; + +namespace StellaOps.Feedser.Source.CertCc.Tests.Internal; + +public sealed class CertCcMapperTests +{ + private static readonly DateTimeOffset PublishedAt = DateTimeOffset.Parse("2025-10-03T11:35:31Z", CultureInfo.InvariantCulture); + + [Fact] + public void Map_ProducesCanonicalAdvisoryWithVendorPrimitives() + { + const string vendorStatement = + "The issue is confirmed, and here is the patch list\n\n" + + "V3912/V3910/V2962/V1000B\t4.4.3.6/4.4.5.1\n" + + "V2927/V2865/V2866\t4.5.1\n" + + "V2765/V2766/V2763/V2135\t4.5.1"; + + var vendor = new CertCcVendorDto( + "DrayTek Corporation", + ContactDate: PublishedAt.AddDays(-10), + StatementDate: PublishedAt.AddDays(-5), + Updated: PublishedAt, + Statement: vendorStatement, + Addendum: null, + References: new[] { "https://www.draytek.com/support/resources?type=version" }); + + var vendorStatus = new CertCcVendorStatusDto( + Vendor: "DrayTek Corporation", + CveId: "CVE-2025-10547", + Status: "Affected", + Statement: null, + References: Array.Empty(), + DateAdded: PublishedAt, + DateUpdated: PublishedAt); + + var vulnerability = new CertCcVulnerabilityDto( + CveId: "CVE-2025-10547", + Description: null, + DateAdded: PublishedAt, + DateUpdated: PublishedAt); + + var metadata = new CertCcNoteMetadata( + VuId: "VU#294418", + IdNumber: "294418", + Title: "Vigor routers running DrayOS RCE via EasyVPN", + Overview: "Overview", + Summary: "Summary", + Published: PublishedAt, + Updated: PublishedAt.AddMinutes(5), + Created: PublishedAt, + Revision: 2, + CveIds: new[] { "CVE-2025-10547" }, + PublicUrls: new[] + { + "https://www.draytek.com/about/security-advisory/use-of-uninitialized-variable-vulnerabilities/", + "https://www.draytek.com/support/resources?type=version" + }, + PrimaryUrl: "https://www.kb.cert.org/vuls/id/294418/"); + + var dto = new CertCcNoteDto( + metadata, + Vendors: new[] { vendor }, + VendorStatuses: new[] { vendorStatus }, + Vulnerabilities: new[] { vulnerability }); + + var document = new DocumentRecord( + Guid.NewGuid(), + "cert-cc", + "https://www.kb.cert.org/vuls/id/294418/", + PublishedAt, + Sha256: new string('0', 64), + Status: "pending-map", + ContentType: "application/json", + Headers: null, + Metadata: null, + Etag: null, + LastModified: PublishedAt, + GridFsId: null); + + var dtoRecord = new DtoRecord( + Id: Guid.NewGuid(), + DocumentId: document.Id, + SourceName: "cert-cc", + SchemaVersion: "certcc.vince.note.v1", + Payload: new BsonDocument(), + ValidatedAt: PublishedAt.AddMinutes(1)); + + var advisory = CertCcMapper.Map(dto, document, dtoRecord, "cert-cc"); + + Assert.Equal("certcc/vu-294418", advisory.AdvisoryKey); + Assert.Contains("VU#294418", advisory.Aliases); + Assert.Contains("CVE-2025-10547", advisory.Aliases); + Assert.Equal("en", advisory.Language); + Assert.Equal(PublishedAt, advisory.Published); + + Assert.Contains(advisory.References, reference => reference.Url.Contains("/vuls/id/294418", StringComparison.OrdinalIgnoreCase)); + + var affected = Assert.Single(advisory.AffectedPackages); + Assert.Equal("vendor", affected.Type); + Assert.Equal("DrayTek Corporation", affected.Identifier); + Assert.Contains(affected.Statuses, status => status.Status == AffectedPackageStatusCatalog.Affected); + + var range = Assert.Single(affected.VersionRanges); + Assert.NotNull(range.Primitives); + Assert.NotNull(range.Primitives!.VendorExtensions); + Assert.Contains(range.Primitives.VendorExtensions!, kvp => kvp.Key == "certcc.vendor.patches"); + + Assert.NotEmpty(affected.NormalizedVersions); + Assert.Contains(affected.NormalizedVersions, rule => rule.Scheme == "certcc.vendor" && rule.Value == "4.5.1"); + } +} diff --git a/src/StellaOps.Feedser.Source.CertCc.Tests/Internal/CertCcSummaryPlannerTests.cs b/src/StellaOps.Feedser.Source.CertCc.Tests/Internal/CertCcSummaryPlannerTests.cs new file mode 100644 index 00000000..0e72c3f0 --- /dev/null +++ b/src/StellaOps.Feedser.Source.CertCc.Tests/Internal/CertCcSummaryPlannerTests.cs @@ -0,0 +1,95 @@ +using System; +using System.Linq; +using Microsoft.Extensions.Options; +using StellaOps.Feedser.Source.CertCc.Configuration; +using StellaOps.Feedser.Source.CertCc.Internal; +using StellaOps.Feedser.Source.Common.Cursors; +using Xunit; + +namespace StellaOps.Feedser.Source.CertCc.Tests.Internal; + +public sealed class CertCcSummaryPlannerTests +{ + [Fact] + public void CreatePlan_UsesInitialBackfillWindow() + { + var options = Options.Create(new CertCcOptions + { + SummaryWindow = new TimeWindowCursorOptions + { + WindowSize = TimeSpan.FromDays(30), + Overlap = TimeSpan.FromDays(3), + InitialBackfill = TimeSpan.FromDays(120), + MinimumWindowSize = TimeSpan.FromDays(1), + }, + }); + + var timeProvider = new TestTimeProvider(DateTimeOffset.Parse("2025-10-10T12:00:00Z")); + var planner = new CertCcSummaryPlanner(options, timeProvider); + + var plan = planner.CreatePlan(state: null); + + Assert.Equal(DateTimeOffset.Parse("2025-06-12T12:00:00Z"), plan.Window.Start); + Assert.Equal(DateTimeOffset.Parse("2025-07-12T12:00:00Z"), plan.Window.End); + + Assert.Equal(3, plan.Requests.Count); + + var monthly = plan.Requests.Where(r => r.Scope == CertCcSummaryScope.Monthly).ToArray(); + Assert.Collection(monthly, + request => + { + Assert.Equal(2025, request.Year); + Assert.Equal(6, request.Month); + Assert.Equal("https://www.kb.cert.org/vuls/api/2025/06/summary/", request.Uri.AbsoluteUri); + }, + request => + { + Assert.Equal(2025, request.Year); + Assert.Equal(7, request.Month); + Assert.Equal("https://www.kb.cert.org/vuls/api/2025/07/summary/", request.Uri.AbsoluteUri); + }); + + var yearly = plan.Requests.Where(r => r.Scope == CertCcSummaryScope.Yearly).ToArray(); + Assert.Single(yearly); + Assert.Equal(2025, yearly[0].Year); + Assert.Null(yearly[0].Month); + Assert.Equal("https://www.kb.cert.org/vuls/api/2025/summary/", yearly[0].Uri.AbsoluteUri); + + Assert.Equal(plan.Window.End, plan.NextState.LastWindowEnd); + } + + [Fact] + public void CreatePlan_AdvancesWindowRespectingOverlap() + { + var options = Options.Create(new CertCcOptions + { + SummaryWindow = new TimeWindowCursorOptions + { + WindowSize = TimeSpan.FromDays(30), + Overlap = TimeSpan.FromDays(10), + InitialBackfill = TimeSpan.FromDays(90), + MinimumWindowSize = TimeSpan.FromDays(1), + }, + }); + + var timeProvider = new TestTimeProvider(DateTimeOffset.Parse("2025-12-01T00:00:00Z")); + var planner = new CertCcSummaryPlanner(options, timeProvider); + + var first = planner.CreatePlan(null); + var second = planner.CreatePlan(first.NextState); + + Assert.True(second.Window.Start < second.Window.End); + Assert.Equal(first.Window.End - options.Value.SummaryWindow.Overlap, second.Window.Start); + } + + private sealed class TestTimeProvider : TimeProvider + { + private DateTimeOffset _now; + + public TestTimeProvider(DateTimeOffset now) => _now = now; + + public override DateTimeOffset GetUtcNow() => _now; + + public void Advance(TimeSpan delta) => _now = _now.Add(delta); + } +} diff --git a/src/StellaOps.Feedser.Source.CertCc.Tests/Internal/CertCcVendorStatementParserTests.cs b/src/StellaOps.Feedser.Source.CertCc.Tests/Internal/CertCcVendorStatementParserTests.cs new file mode 100644 index 00000000..582a198a --- /dev/null +++ b/src/StellaOps.Feedser.Source.CertCc.Tests/Internal/CertCcVendorStatementParserTests.cs @@ -0,0 +1,31 @@ +using System.Linq; +using StellaOps.Feedser.Source.CertCc.Internal; +using Xunit; + +namespace StellaOps.Feedser.Source.CertCc.Tests.Internal; + +public sealed class CertCcVendorStatementParserTests +{ + [Fact] + public void Parse_ReturnsPatchesForTabDelimitedList() + { + const string statement = + "V3912/V3910/V2962/V1000B\t4.4.3.6/4.4.5.1\n" + + "V2927/V2865/V2866\t4.5.1\n" + + "V2765/V2766/V2763/V2135\t4.5.1"; + + var patches = CertCcVendorStatementParser.Parse(statement); + + Assert.Equal(11, patches.Count); + Assert.Contains(patches, patch => patch.Product == "V3912" && patch.Version == "4.4.3.6"); + Assert.Contains(patches, patch => patch.Product == "V2962" && patch.Version == "4.4.5.1"); + Assert.Equal(7, patches.Count(patch => patch.Version == "4.5.1")); + } + + [Fact] + public void Parse_ReturnsEmptyWhenStatementMissing() + { + var patches = CertCcVendorStatementParser.Parse(null); + Assert.Empty(patches); + } +} diff --git a/src/StellaOps.Feedser.Source.CertCc.Tests/StellaOps.Feedser.Source.CertCc.Tests.csproj b/src/StellaOps.Feedser.Source.CertCc.Tests/StellaOps.Feedser.Source.CertCc.Tests.csproj new file mode 100644 index 00000000..8464ae74 --- /dev/null +++ b/src/StellaOps.Feedser.Source.CertCc.Tests/StellaOps.Feedser.Source.CertCc.Tests.csproj @@ -0,0 +1,16 @@ + + + net10.0 + enable + enable + + + + + + + + PreserveNewest + + + diff --git a/src/StellaOps.Feedser.Source.CertCc/AGENTS.md b/src/StellaOps.Feedser.Source.CertCc/AGENTS.md new file mode 100644 index 00000000..453f0c10 --- /dev/null +++ b/src/StellaOps.Feedser.Source.CertCc/AGENTS.md @@ -0,0 +1,38 @@ +# AGENTS +## Role +Implement the CERT/CC (Carnegie Mellon CERT Coordination Center) advisory connector so Feedser can ingest US CERT coordination bulletins. + +## Scope +- Identify CERT/CC advisory publication format (VU#, blog, RSS, JSON) and define fetch cadence/windowing. +- Implement fetch, parse, and mapping jobs with cursor persistence and dedupe. +- Normalise advisory content (summary, impacted vendors, products, recommended mitigations, CVEs). +- Produce canonical `Advisory` objects including aliases, references, affected packages, and range primitive metadata. +- Supply fixtures and deterministic regression tests. + +## Participants +- `Source.Common` (HTTP/fetch utilities, DTO storage). +- `Storage.Mongo` (raw/document/DTO/advisory stores and state). +- `Feedser.Models` (canonical structures). +- `Feedser.Testing` (integration tests and snapshots). + +## Interfaces & Contracts +- Job kinds: `certcc:fetch`, `certcc:parse`, `certcc:map`. +- Persist upstream caching metadata (ETag/Last-Modified) when available. +- Aliases should capture CERT/CC VU IDs and referenced CVEs. + +## In/Out of scope +In scope: +- End-to-end connector with range primitive instrumentation and telemetry. + +Out of scope: +- ICS-CERT alerts (handled by dedicated connector) or blog posts unrelated to advisories. + +## Observability & Security Expectations +- Log fetch and mapping statistics; surface failures with backoff. +- Sanitise HTML sources before persistence. +- Respect upstream throttling via retry/backoff. + +## Tests +- Add `StellaOps.Feedser.Source.CertCc.Tests` to cover fetch/parse/map with canned fixtures. +- Snapshot canonical advisories and support UPDATE flag for regeneration. +- Ensure deterministic ordering and timestamp normalisation. diff --git a/src/StellaOps.Feedser.Source.CertCc/CertCcConnector.cs b/src/StellaOps.Feedser.Source.CertCc/CertCcConnector.cs new file mode 100644 index 00000000..4cd43190 --- /dev/null +++ b/src/StellaOps.Feedser.Source.CertCc/CertCcConnector.cs @@ -0,0 +1,124 @@ +using System; +using System.Collections.Generic; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Options; +using StellaOps.Feedser.Source.CertCc.Configuration; +using StellaOps.Feedser.Source.CertCc.Internal; +using StellaOps.Feedser.Source.Common; +using StellaOps.Feedser.Source.Common.Fetch; +using StellaOps.Feedser.Source.Common.Cursors; +using StellaOps.Feedser.Storage.Mongo; +using StellaOps.Feedser.Storage.Mongo.Documents; +using StellaOps.Plugin; + +namespace StellaOps.Feedser.Source.CertCc; + +public sealed class CertCcConnector : IFeedConnector +{ + private readonly CertCcSummaryPlanner _summaryPlanner; + private readonly SourceFetchService _fetchService; + private readonly IDocumentStore _documentStore; + private readonly ISourceStateRepository _stateRepository; + private readonly CertCcOptions _options; + private readonly TimeProvider _timeProvider; + private readonly ILogger _logger; + + public CertCcConnector( + CertCcSummaryPlanner summaryPlanner, + SourceFetchService fetchService, + IDocumentStore documentStore, + ISourceStateRepository stateRepository, + IOptions options, + TimeProvider? timeProvider, + ILogger logger) + { + _summaryPlanner = summaryPlanner ?? throw new ArgumentNullException(nameof(summaryPlanner)); + _fetchService = fetchService ?? throw new ArgumentNullException(nameof(fetchService)); + _documentStore = documentStore ?? throw new ArgumentNullException(nameof(documentStore)); + _stateRepository = stateRepository ?? throw new ArgumentNullException(nameof(stateRepository)); + _options = (options ?? throw new ArgumentNullException(nameof(options))).Value ?? throw new ArgumentNullException(nameof(options)); + _options.Validate(); + _timeProvider = timeProvider ?? TimeProvider.System; + _logger = logger ?? throw new ArgumentNullException(nameof(logger)); + } + + public string SourceName => CertCcConnectorPlugin.SourceName; + + public async Task FetchAsync(IServiceProvider services, CancellationToken cancellationToken) + { + var cursor = await GetCursorAsync(cancellationToken).ConfigureAwait(false); + var plan = _summaryPlanner.CreatePlan(cursor.SummaryState); + if (plan.Requests.Count == 0) + { + await UpdateCursorAsync(cursor.WithSummaryState(plan.NextState).WithLastRun(_timeProvider.GetUtcNow()), cancellationToken).ConfigureAwait(false); + return; + } + + foreach (var request in plan.Requests) + { + cancellationToken.ThrowIfCancellationRequested(); + + try + { + var uri = request.Uri; + var existing = await _documentStore.FindBySourceAndUriAsync(SourceName, uri.ToString(), cancellationToken).ConfigureAwait(false); + var metadata = new Dictionary(StringComparer.OrdinalIgnoreCase) + { + ["certcc.scope"] = request.Scope.ToString().ToLowerInvariant(), + ["certcc.year"] = request.Year.ToString("D4"), + }; + + if (request.Month.HasValue) + { + metadata["certcc.month"] = request.Month.Value.ToString("D2"); + } + + var fetchRequest = new SourceFetchRequest(CertCcOptions.HttpClientName, SourceName, uri) + { + Metadata = metadata, + AcceptHeaders = new[] { "application/json" }, + ETag = existing?.Etag, + LastModified = existing?.LastModified, + }; + + var result = await _fetchService.FetchAsync(fetchRequest, cancellationToken).ConfigureAwait(false); + if (result.IsNotModified) + { + _logger.LogDebug("CERT/CC summary {Uri} returned 304 Not Modified", uri); + } + } + catch (Exception ex) + { + _logger.LogError(ex, "CERT/CC summary fetch failed for {Uri}", request.Uri); + await _stateRepository.MarkFailureAsync(SourceName, _timeProvider.GetUtcNow(), TimeSpan.FromMinutes(5), ex.Message, cancellationToken).ConfigureAwait(false); + throw; + } + } + + var updatedCursor = cursor + .WithSummaryState(plan.NextState) + .WithLastRun(_timeProvider.GetUtcNow()); + + await UpdateCursorAsync(updatedCursor, cancellationToken).ConfigureAwait(false); + } + + public Task ParseAsync(IServiceProvider services, CancellationToken cancellationToken) + => Task.CompletedTask; + + public Task MapAsync(IServiceProvider services, CancellationToken cancellationToken) + => Task.CompletedTask; + + private async Task GetCursorAsync(CancellationToken cancellationToken) + { + var record = await _stateRepository.TryGetAsync(SourceName, cancellationToken).ConfigureAwait(false); + return CertCcCursor.FromBson(record?.Cursor); + } + + private async Task UpdateCursorAsync(CertCcCursor cursor, CancellationToken cancellationToken) + { + var document = cursor.ToBsonDocument(); + await _stateRepository.UpdateCursorAsync(SourceName, document, _timeProvider.GetUtcNow(), cancellationToken).ConfigureAwait(false); + } +} diff --git a/src/StellaOps.Feedser.Source.CertCc/CertCcConnectorPlugin.cs b/src/StellaOps.Feedser.Source.CertCc/CertCcConnectorPlugin.cs new file mode 100644 index 00000000..468a1b20 --- /dev/null +++ b/src/StellaOps.Feedser.Source.CertCc/CertCcConnectorPlugin.cs @@ -0,0 +1,21 @@ +using System; +using Microsoft.Extensions.DependencyInjection; +using StellaOps.Plugin; + +namespace StellaOps.Feedser.Source.CertCc; + +public sealed class CertCcConnectorPlugin : IConnectorPlugin +{ + public const string SourceName = "cert-cc"; + + public string Name => SourceName; + + public bool IsAvailable(IServiceProvider services) + => services.GetService() is not null; + + public IFeedConnector Create(IServiceProvider services) + { + ArgumentNullException.ThrowIfNull(services); + return services.GetRequiredService(); + } +} diff --git a/src/StellaOps.Feedser.Source.CertCc/CertCcDependencyInjectionRoutine.cs b/src/StellaOps.Feedser.Source.CertCc/CertCcDependencyInjectionRoutine.cs new file mode 100644 index 00000000..bf09ce4a --- /dev/null +++ b/src/StellaOps.Feedser.Source.CertCc/CertCcDependencyInjectionRoutine.cs @@ -0,0 +1,50 @@ +using System; +using Microsoft.Extensions.Configuration; +using Microsoft.Extensions.DependencyInjection; +using StellaOps.DependencyInjection; +using StellaOps.Feedser.Core.Jobs; +using StellaOps.Feedser.Source.CertCc.Configuration; + +namespace StellaOps.Feedser.Source.CertCc; + +public sealed class CertCcDependencyInjectionRoutine : IDependencyInjectionRoutine +{ + private const string ConfigurationSection = "feedser:sources:cert-cc"; + + public IServiceCollection Register(IServiceCollection services, IConfiguration configuration) + { + ArgumentNullException.ThrowIfNull(services); + ArgumentNullException.ThrowIfNull(configuration); + + services.AddCertCcConnector(options => + { + configuration.GetSection(ConfigurationSection).Bind(options); + options.Validate(); + }); + + services.AddTransient(); + + services.PostConfigure(options => + { + EnsureJob(options, CertCcJobKinds.Fetch, typeof(CertCcFetchJob)); + }); + + return services; + } + + private static void EnsureJob(JobSchedulerOptions options, string kind, Type jobType) + { + if (options.Definitions.ContainsKey(kind)) + { + return; + } + + options.Definitions[kind] = new JobDefinition( + kind, + jobType, + options.DefaultTimeout, + options.DefaultLeaseDuration, + CronExpression: null, + Enabled: true); + } +} diff --git a/src/StellaOps.Feedser.Source.CertCc/CertCcServiceCollectionExtensions.cs b/src/StellaOps.Feedser.Source.CertCc/CertCcServiceCollectionExtensions.cs new file mode 100644 index 00000000..fdcb4070 --- /dev/null +++ b/src/StellaOps.Feedser.Source.CertCc/CertCcServiceCollectionExtensions.cs @@ -0,0 +1,36 @@ +using System; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.DependencyInjection.Extensions; +using Microsoft.Extensions.Options; +using StellaOps.Feedser.Source.CertCc.Configuration; +using StellaOps.Feedser.Source.CertCc.Internal; +using StellaOps.Feedser.Source.Common.Http; + +namespace StellaOps.Feedser.Source.CertCc; + +public static class CertCcServiceCollectionExtensions +{ + public static IServiceCollection AddCertCcConnector(this IServiceCollection services, Action configure) + { + ArgumentNullException.ThrowIfNull(services); + ArgumentNullException.ThrowIfNull(configure); + + services.AddOptions() + .Configure(configure) + .PostConfigure(static options => options.Validate()); + + services.AddSourceHttpClient(CertCcOptions.HttpClientName, static (sp, clientOptions) => + { + var options = sp.GetRequiredService>().Value; + clientOptions.BaseAddress = options.BaseApiUri; + clientOptions.UserAgent = "StellaOps.Feedser.CertCc/1.0"; + clientOptions.Timeout = TimeSpan.FromSeconds(20); + clientOptions.AllowedHosts.Clear(); + clientOptions.AllowedHosts.Add(options.BaseApiUri.Host); + }); + + services.TryAddSingleton(); + services.AddTransient(); + return services; + } +} diff --git a/src/StellaOps.Feedser.Source.CertCc/Configuration/CertCcOptions.cs b/src/StellaOps.Feedser.Source.CertCc/Configuration/CertCcOptions.cs new file mode 100644 index 00000000..c114d9cd --- /dev/null +++ b/src/StellaOps.Feedser.Source.CertCc/Configuration/CertCcOptions.cs @@ -0,0 +1,54 @@ +using System; +using StellaOps.Feedser.Source.Common.Cursors; + +namespace StellaOps.Feedser.Source.CertCc.Configuration; + +/// +/// Connector options governing CERT/CC fetch cadence and API endpoints. +/// +public sealed class CertCcOptions +{ + public const string HttpClientName = "certcc"; + + /// + /// Root URI for the VINCE Vulnerability Notes API (must end with a slash). + /// + public Uri BaseApiUri { get; set; } = new("https://www.kb.cert.org/vuls/api/", UriKind.Absolute); + + /// + /// Sliding window settings controlling which summary endpoints are requested. + /// + public TimeWindowCursorOptions SummaryWindow { get; set; } = new() + { + WindowSize = TimeSpan.FromDays(30), + Overlap = TimeSpan.FromDays(3), + InitialBackfill = TimeSpan.FromDays(365), + MinimumWindowSize = TimeSpan.FromDays(1), + }; + + /// + /// Maximum number of monthly summary endpoints to request in a single plan. + /// + public int MaxMonthlySummaries { get; set; } = 6; + + public void Validate() + { + if (BaseApiUri is null || !BaseApiUri.IsAbsoluteUri) + { + throw new InvalidOperationException("CertCcOptions.BaseApiUri must be an absolute URI."); + } + + if (!BaseApiUri.AbsoluteUri.EndsWith("/", StringComparison.Ordinal)) + { + throw new InvalidOperationException("CertCcOptions.BaseApiUri must end with a trailing slash."); + } + + SummaryWindow ??= new TimeWindowCursorOptions(); + SummaryWindow.EnsureValid(); + + if (MaxMonthlySummaries <= 0) + { + throw new InvalidOperationException("CertCcOptions.MaxMonthlySummaries must be positive."); + } + } +} diff --git a/src/StellaOps.Feedser.Source.CertCc/FEEDCONN-CERTCC-02-009_PLAN.md b/src/StellaOps.Feedser.Source.CertCc/FEEDCONN-CERTCC-02-009_PLAN.md new file mode 100644 index 00000000..16d5dd34 --- /dev/null +++ b/src/StellaOps.Feedser.Source.CertCc/FEEDCONN-CERTCC-02-009_PLAN.md @@ -0,0 +1,59 @@ +# FEEDCONN-CERTCC-02-009 – VINCE Detail & Map Reintegration Plan + +- **Author:** BE-Conn-CERTCC (current on-call) +- **Date:** 2025-10-11 +- **Scope:** Restore VINCE detail parsing and canonical mapping in Feedser without destabilising downstream Merge/Export pipelines. + +## 1. Current State Snapshot (2025-10-11) + +- ✅ Fetch pipeline, VINCE summary planner, and detail queue are live; documents land with `DocumentStatuses.PendingParse`. +- ✅ DTO aggregate (`CertCcNoteDto`) plus mapper emit vendor-centric `normalizedVersions` (`scheme=certcc.vendor`) and provenance aligned with `src/StellaOps.Feedser.Models/PROVENANCE_GUIDELINES.md`. +- ✅ Regression coverage exists for fetch/parse/map flows (`CertCcConnectorSnapshotTests`), but snapshot regeneration is gated on harness refresh (FEEDCONN-CERTCC-02-007) and QA handoff (FEEDCONN-CERTCC-02-008). +- ⚠️ Parse/map jobs are not scheduled; production still operates in fetch-only mode. +- ⚠️ Downstream Merge team is finalising normalized range ingestion per `src/FASTER_MODELING_AND_NORMALIZATION.md`; we must avoid publishing canonical records until they certify compatibility. + +## 2. Required Dependencies & Coordinated Tasks + +| Dependency | Owner(s) | Blocking Condition | Handshake | +|------------|----------|--------------------|-----------| +| FEEDCONN-CERTCC-02-004 (Canonical mapping & range primitives hardening) | BE-Conn-CERTCC + Models | Ensure mapper emits deterministic `normalizedVersions` array and provenance field masks | Daily sync with Models/Merge leads; share fixture diff before each enablement phase | +| FEEDCONN-CERTCC-02-007 (Connector test harness remediation) | BE-Conn-CERTCC, QA | Restore `AddSourceCommon` harness + canned VINCE fixtures so we can shadow-run parse/map | Required before Phase 1 | +| FEEDCONN-CERTCC-02-008 (Snapshot coverage handoff) | QA | Snapshot refresh process green to surface regressions | Required before Phase 2 | +| FEEDCONN-CERTCC-02-010 (Partial-detail graceful degradation) | BE-Conn-CERTCC | Resiliency for missing VINCE endpoints to avoid job wedging after reintegration | Should land before Phase 2 cutover | + +## 3. Phased Rollout Plan + +| Phase | Window (UTC) | Actions | Success Signals | Rollback | +|-------|--------------|---------|-----------------|----------| +| **0 – Pre-flight validation** | 2025-10-11 → 2025-10-12 | • Finish FEEDCONN-CERTCC-02-007 harness fixes and regenerate fixtures.
    • Run `dotnet test src/StellaOps.Feedser.Source.CertCc.Tests` with `UPDATE_CERTCC_FIXTURES=0` to confirm deterministic baselines.
    • Generate sample advisory batch (`dotnet test … --filter SnapshotSmoke`) and deliver JSON diff to Merge for schema verification (`normalizedVersions[].scheme == certcc.vendor`, provenance masks populated). | • Harness tests green locally and in CI.
    • Merge sign-off that sample advisories conform to `FASTER_MODELING_AND_NORMALIZATION.md`. | N/A (no production enablement yet). | +| **1 – Shadow parse/map in staging** | Target start 2025-10-13 | • Register `source:cert-cc:parse` and `source:cert-cc:map` jobs, but gate them behind new config flag `feedser:sources:cert-cc:enableDetailMapping` (default `false`).
    • Deploy (restart required for options rebinding), enable flag, and point connector at staging Mongo with isolated collection (`advisories_certcc_shadow`).
    • Run connector for ≥2 cycles; compare advisory counts vs. fetch-only baseline and validate `feedser.range.primitives` metrics include `scheme=certcc.vendor`. | • No uncaught exceptions in staging logs.
    • Shadow advisories match expected vendor counts (±5%).
    • `certcc.summary.fetch.*` + new `certcc.map.duration.ms` metrics stable. | Disable flag; staging returns to fetch-only. No production impact. | +| **2 – Controlled production enablement** | Target start 2025-10-14 | • Redeploy production with flag enabled, start with job concurrency `1`, and reduce `MaxNotesPerFetch` to 5 for first 24 h.
    • Observe metrics dashboards hourly (fetch/map latency, pending queues, Mongo write throughput).
    • QA to replay latest snapshots and confirm no deterministic drift.
    • Publish advisory sample (top 10 changed docs) to Merge Slack channel for validation. | • Pending parse/mapping queues drain within expected SLA (<30 min).
    • No increase in merge dedupe anomalies.
    • Mongo writes stay within 10% of baseline. | Toggle flag off, re-run fetch-only. Clear `pendingMappings` via connector cursor reset if stuck. | +| **3 – Full production & cleanup** | Target start 2025-10-15 | • Restore `MaxNotesPerFetch` to configured default (20).
    • Remove temporary throttles and leave flag enabled by default.
    • Update `README.md` rollout notes; close FEEDCONN-CERTCC-02-009.
    • Kick off post-merge audit with Merge to ensure new advisories dedupe with other sources. | • Stable operations for ≥48 h, no degradation alerts.
    • Merge confirms conflict resolver behaviour unchanged. | If regression detected, revert to Phase 2 state or disable jobs; retain plan for reuse. | + +## 4. Monitoring & Validation Checklist + +- Dashboards: `certcc.*` meters (plan, summary fetch, detail fetch) plus `feedser.range.primitives` with tag `scheme=certcc.vendor`. +- Logs: ensure Parse/Map jobs emit `correlationId` aligned with fetch events for traceability. +- Data QA: run `tools/dump_advisory` against two VINCE notes (one multi-vendor, one single-vendor) every phase to spot-check normalized versions ordering and provenance. +- Storage: verify Mongo TTL/size for `raw_documents` and `dtos`—detail payload volume increases by ~3× when mapping resumes. + +## 5. Rollback / Contingency Playbook + +1. Disable `feedser:sources:cert-cc:enableDetailMapping` flag (and optionally set `MaxNotesPerFetch=0` for a single cycle) to halt new detail ingestion. +2. Run connector once to update cursor; verify `pendingMappings` drains. +3. If advisories already persisted, coordinate with Merge to soft-delete affected `certcc/*` advisories by advisory key hash (no schema rollback required). +4. Re-run Phase 1 shadow validation before retrying. + +## 6. Communication Cadence + +- Daily check-in with Models/Merge leads (09:30 EDT) to surface normalizedVersions/provenance diffs. +- Post-phase reports in `#feedser-certcc` Slack channel summarising metrics, advisory counts, and outstanding issues. +- Escalate blockers >12 h via Runbook SEV-3 path and annotate `TASKS.md`. + +## 7. Open Questions / Next Actions + +- [ ] Confirm whether Merge requires additional provenance field masks before Phase 2 (waiting on feedback from 2025-10-11 sample). +- [ ] Decide if CSAF endpoint ingestion (optional) should piggyback on Phase 3 or stay deferred. +- [ ] Validate that FEEDCONN-CERTCC-02-010 coverage handles mixed 200/404 VINCE endpoints during partial outages. + +Once Dependencies (Section 2) are cleared and Phase 3 completes, update `src/StellaOps.Feedser.Source.CertCc/TASKS.md` and close FEEDCONN-CERTCC-02-009. diff --git a/src/StellaOps.Feedser.Source.CertCc/Internal/CertCcCursor.cs b/src/StellaOps.Feedser.Source.CertCc/Internal/CertCcCursor.cs new file mode 100644 index 00000000..16db9a07 --- /dev/null +++ b/src/StellaOps.Feedser.Source.CertCc/Internal/CertCcCursor.cs @@ -0,0 +1,58 @@ +using MongoDB.Bson; +using StellaOps.Feedser.Source.Common.Cursors; + +namespace StellaOps.Feedser.Source.CertCc.Internal; + +internal sealed record CertCcCursor(TimeWindowCursorState SummaryState, DateTimeOffset? LastRun) +{ + public static CertCcCursor Empty { get; } = new(TimeWindowCursorState.Empty, null); + + public BsonDocument ToBsonDocument() + { + var document = new BsonDocument(); + + var summary = new BsonDocument(); + SummaryState.WriteTo(summary, "start", "end"); + document["summary"] = summary; + + if (LastRun.HasValue) + { + document["lastRun"] = LastRun.Value.UtcDateTime; + } + + return document; + } + + public static CertCcCursor FromBson(BsonDocument? document) + { + if (document is null || document.ElementCount == 0) + { + return Empty; + } + + TimeWindowCursorState summaryState = TimeWindowCursorState.Empty; + if (document.TryGetValue("summary", out var summaryValue) && summaryValue is BsonDocument summaryDocument) + { + summaryState = TimeWindowCursorState.FromBsonDocument(summaryDocument, "start", "end"); + } + + DateTimeOffset? lastRun = null; + if (document.TryGetValue("lastRun", out var lastRunValue)) + { + lastRun = lastRunValue.BsonType switch + { + BsonType.DateTime => DateTime.SpecifyKind(lastRunValue.ToUniversalTime(), DateTimeKind.Utc), + BsonType.String when DateTimeOffset.TryParse(lastRunValue.AsString, out var parsed) => parsed.ToUniversalTime(), + _ => null, + }; + } + + return new CertCcCursor(summaryState, lastRun); + } + + public CertCcCursor WithSummaryState(TimeWindowCursorState state) + => this with { SummaryState = state }; + + public CertCcCursor WithLastRun(DateTimeOffset? timestamp) + => this with { LastRun = timestamp }; +} diff --git a/src/StellaOps.Feedser.Source.CertCc/Internal/CertCcDiagnostics.cs b/src/StellaOps.Feedser.Source.CertCc/Internal/CertCcDiagnostics.cs new file mode 100644 index 00000000..874dc192 --- /dev/null +++ b/src/StellaOps.Feedser.Source.CertCc/Internal/CertCcDiagnostics.cs @@ -0,0 +1,90 @@ +using System; +using System.Collections.Generic; +using System.Diagnostics.Metrics; +using StellaOps.Feedser.Source.Common.Cursors; + +namespace StellaOps.Feedser.Source.CertCc.Internal; + +/// +/// Emits CERT/CC-specific telemetry for summary planning and fetch activity. +/// +public sealed class CertCcDiagnostics : IDisposable +{ + private const string MeterName = "StellaOps.Feedser.Source.CertCc"; + private const string MeterVersion = "1.0.0"; + + private readonly Meter _meter; + private readonly Counter _planWindows; + private readonly Counter _planRequests; + private readonly Histogram _planWindowDays; + private readonly Counter _summaryFetchAttempts; + private readonly Counter _summaryFetchSuccess; + private readonly Counter _summaryFetchUnchanged; + private readonly Counter _summaryFetchFailures; + + public CertCcDiagnostics() + { + _meter = new Meter(MeterName, MeterVersion); + _planWindows = _meter.CreateCounter( + name: "certcc.plan.windows", + unit: "windows", + description: "Number of summary planning windows evaluated."); + _planRequests = _meter.CreateCounter( + name: "certcc.plan.requests", + unit: "requests", + description: "Total CERT/CC summary endpoints queued by the planner."); + _planWindowDays = _meter.CreateHistogram( + name: "certcc.plan.window_days", + unit: "day", + description: "Duration of each planning window in days."); + _summaryFetchAttempts = _meter.CreateCounter( + name: "certcc.summary.fetch.attempts", + unit: "operations", + description: "Number of VINCE summary fetch attempts."); + _summaryFetchSuccess = _meter.CreateCounter( + name: "certcc.summary.fetch.success", + unit: "operations", + description: "Number of VINCE summary fetches persisted to storage."); + _summaryFetchUnchanged = _meter.CreateCounter( + name: "certcc.summary.fetch.not_modified", + unit: "operations", + description: "Number of VINCE summary fetches returning HTTP 304."); + _summaryFetchFailures = _meter.CreateCounter( + name: "certcc.summary.fetch.failures", + unit: "operations", + description: "Number of VINCE summary fetches that failed after retries."); + } + + public void PlanEvaluated(TimeWindow window, int requestCount) + { + _planWindows.Add(1); + + if (requestCount > 0) + { + _planRequests.Add(requestCount); + } + + var duration = window.Duration; + if (duration > TimeSpan.Zero) + { + _planWindowDays.Record(duration.TotalDays); + } + } + + public void SummaryFetchAttempt(CertCcSummaryScope scope) + => _summaryFetchAttempts.Add(1, ScopeTag(scope)); + + public void SummaryFetchSuccess(CertCcSummaryScope scope) + => _summaryFetchSuccess.Add(1, ScopeTag(scope)); + + public void SummaryFetchUnchanged(CertCcSummaryScope scope) + => _summaryFetchUnchanged.Add(1, ScopeTag(scope)); + + public void SummaryFetchFailure(CertCcSummaryScope scope) + => _summaryFetchFailures.Add(1, ScopeTag(scope)); + + private static KeyValuePair ScopeTag(CertCcSummaryScope scope) + => new("scope", scope.ToString().ToLowerInvariant()); + + public void Dispose() => _meter.Dispose(); +} diff --git a/src/StellaOps.Feedser.Source.CertCc/Internal/CertCcMapper.cs b/src/StellaOps.Feedser.Source.CertCc/Internal/CertCcMapper.cs new file mode 100644 index 00000000..d4dfc714 --- /dev/null +++ b/src/StellaOps.Feedser.Source.CertCc/Internal/CertCcMapper.cs @@ -0,0 +1,607 @@ +using System; +using System.Collections.Generic; +using System.Globalization; +using System.Linq; +using System.Net; +using System.Text.RegularExpressions; +using StellaOps.Feedser.Models; +using StellaOps.Feedser.Storage.Mongo.Documents; +using StellaOps.Feedser.Storage.Mongo.Dtos; + +namespace StellaOps.Feedser.Source.CertCc.Internal; + +internal static class CertCcMapper +{ + private const string AdvisoryPrefix = "certcc"; + private const string VendorNormalizedVersionScheme = "certcc.vendor"; + + public static Advisory Map( + CertCcNoteDto dto, + DocumentRecord document, + DtoRecord dtoRecord, + string sourceName) + { + ArgumentNullException.ThrowIfNull(dto); + ArgumentNullException.ThrowIfNull(document); + ArgumentNullException.ThrowIfNull(dtoRecord); + ArgumentException.ThrowIfNullOrEmpty(sourceName); + + var recordedAt = dtoRecord.ValidatedAt.ToUniversalTime(); + var fetchedAt = document.FetchedAt.ToUniversalTime(); + + var metadata = dto.Metadata ?? CertCcNoteMetadata.Empty; + + var advisoryKey = BuildAdvisoryKey(metadata); + var title = string.IsNullOrWhiteSpace(metadata.Title) ? advisoryKey : metadata.Title.Trim(); + var summary = ExtractSummary(metadata); + + var aliases = BuildAliases(dto).ToArray(); + var references = BuildReferences(dto, metadata, sourceName, recordedAt).ToArray(); + var affectedPackages = BuildAffectedPackages(dto, metadata, sourceName, recordedAt).ToArray(); + + var provenance = new[] + { + new AdvisoryProvenance(sourceName, "document", document.Uri, fetchedAt), + new AdvisoryProvenance(sourceName, "map", metadata.VuId ?? metadata.IdNumber ?? advisoryKey, recordedAt), + }; + + return new Advisory( + advisoryKey, + title, + summary, + language: "en", + metadata.Published?.ToUniversalTime(), + metadata.Updated?.ToUniversalTime(), + severity: null, + exploitKnown: false, + aliases, + references, + affectedPackages, + cvssMetrics: Array.Empty(), + provenance); + } + + private static string BuildAdvisoryKey(CertCcNoteMetadata metadata) + { + if (metadata is null) + { + return $"{AdvisoryPrefix}/{Guid.NewGuid():N}"; + } + + var vuKey = NormalizeVuId(metadata.VuId); + if (vuKey.Length > 0) + { + return $"{AdvisoryPrefix}/{vuKey}"; + } + + var id = SanitizeToken(metadata.IdNumber); + if (id.Length > 0) + { + return $"{AdvisoryPrefix}/vu-{id}"; + } + + return $"{AdvisoryPrefix}/{Guid.NewGuid():N}"; + } + + private static string NormalizeVuId(string? value) + { + if (string.IsNullOrWhiteSpace(value)) + { + return string.Empty; + } + + var digits = new string(value.Where(char.IsDigit).ToArray()); + if (digits.Length > 0) + { + return $"vu-{digits}"; + } + + var sanitized = value.Trim().ToLowerInvariant(); + sanitized = sanitized.Replace("vu#", "vu-", StringComparison.OrdinalIgnoreCase); + sanitized = sanitized.Replace('#', '-'); + sanitized = sanitized.Replace(' ', '-'); + + return SanitizeToken(sanitized); + } + + private static string SanitizeToken(string? value) + { + if (string.IsNullOrWhiteSpace(value)) + { + return string.Empty; + } + + var trimmed = value.Trim(); + var filtered = new string(trimmed + .Select(ch => char.IsLetterOrDigit(ch) || ch is '-' or '_' ? ch : '-') + .ToArray()); + + return filtered.Trim('-').ToLowerInvariant(); + } + + private static readonly Regex HtmlTagRegex = new("<[^>]+>", RegexOptions.Compiled | RegexOptions.CultureInvariant); + private static readonly Regex WhitespaceRegex = new("[ \t\f\r]+", RegexOptions.Compiled | RegexOptions.CultureInvariant); + private static readonly Regex ParagraphRegex = new("<\\s*/?\\s*p[^>]*>", RegexOptions.Compiled | RegexOptions.IgnoreCase | RegexOptions.CultureInvariant); + + private static string? ExtractSummary(CertCcNoteMetadata metadata) + { + if (metadata is null) + { + return null; + } + + var summary = string.IsNullOrWhiteSpace(metadata.Summary) ? metadata.Overview : metadata.Summary; + if (string.IsNullOrWhiteSpace(summary)) + { + return null; + } + + return HtmlToPlainText(summary); + } + + private static string HtmlToPlainText(string html) + { + if (string.IsNullOrWhiteSpace(html)) + { + return string.Empty; + } + + var normalized = html + .Replace("
    ", "\n", StringComparison.OrdinalIgnoreCase) + .Replace("
    ", "\n", StringComparison.OrdinalIgnoreCase) + .Replace("
    ", "\n", StringComparison.OrdinalIgnoreCase) + .Replace("
  • ", "\n", StringComparison.OrdinalIgnoreCase) + .Replace("
    • ", "\n", StringComparison.OrdinalIgnoreCase); + + normalized = ParagraphRegex.Replace(normalized, "\n"); + + var withoutTags = HtmlTagRegex.Replace(normalized, " "); + var decoded = WebUtility.HtmlDecode(withoutTags) ?? string.Empty; + var collapsedSpaces = WhitespaceRegex.Replace(decoded, " "); + var collapsedNewlines = Regex.Replace(collapsedSpaces, "\n{2,}", "\n", RegexOptions.Compiled); + return collapsedNewlines.Trim(); + } + + private static IEnumerable BuildAliases(CertCcNoteDto dto) + { + var aliases = new HashSet(StringComparer.OrdinalIgnoreCase); + + var metadata = dto.Metadata ?? CertCcNoteMetadata.Empty; + + if (!string.IsNullOrWhiteSpace(metadata.VuId)) + { + aliases.Add(metadata.VuId.Trim()); + } + + if (!string.IsNullOrWhiteSpace(metadata.IdNumber)) + { + aliases.Add($"VU#{metadata.IdNumber.Trim()}"); + } + + foreach (var cve in metadata.CveIds ?? Array.Empty()) + { + if (string.IsNullOrWhiteSpace(cve)) + { + continue; + } + + aliases.Add(cve.Trim()); + } + + foreach (var vulnerability in dto.Vulnerabilities ?? Array.Empty()) + { + if (string.IsNullOrWhiteSpace(vulnerability.CveId)) + { + continue; + } + + aliases.Add(vulnerability.CveId.Trim()); + } + + return aliases.OrderBy(static alias => alias, StringComparer.OrdinalIgnoreCase); + } + + private static IEnumerable BuildReferences( + CertCcNoteDto dto, + CertCcNoteMetadata metadata, + string sourceName, + DateTimeOffset recordedAt) + { + var references = new List(); + var canonicalUri = !string.IsNullOrWhiteSpace(metadata.PrimaryUrl) + ? metadata.PrimaryUrl! + : (string.IsNullOrWhiteSpace(metadata.IdNumber) + ? "https://www.kb.cert.org/vuls/" + : $"https://www.kb.cert.org/vuls/id/{metadata.IdNumber.Trim()}/"); + + var provenance = new AdvisoryProvenance(sourceName, "reference", canonicalUri, recordedAt); + + TryAddReference(references, canonicalUri, "advisory", "certcc.note", null, provenance); + + foreach (var url in metadata.PublicUrls ?? Array.Empty()) + { + TryAddReference(references, url, "reference", "certcc.public", null, provenance); + } + + foreach (var vendor in dto.Vendors ?? Array.Empty()) + { + foreach (var url in vendor.References ?? Array.Empty()) + { + TryAddReference(references, url, "reference", "certcc.vendor", vendor.Vendor, provenance); + } + + var statementText = vendor.Statement ?? string.Empty; + var patches = CertCcVendorStatementParser.Parse(statementText); + foreach (var patch in patches) + { + if (!string.IsNullOrWhiteSpace(patch.RawLine) && TryFindEmbeddedUrl(patch.RawLine!, out var rawUrl)) + { + TryAddReference(references, rawUrl, "reference", "certcc.vendor.statement", vendor.Vendor, provenance); + } + } + } + + foreach (var status in dto.VendorStatuses ?? Array.Empty()) + { + foreach (var url in status.References ?? Array.Empty()) + { + TryAddReference(references, url, "reference", "certcc.vendor.status", status.Vendor, provenance); + } + + if (!string.IsNullOrWhiteSpace(status.Statement) && TryFindEmbeddedUrl(status.Statement!, out var embedded)) + { + TryAddReference(references, embedded, "reference", "certcc.vendor.status", status.Vendor, provenance); + } + } + + return references + .GroupBy(static reference => reference.Url, StringComparer.OrdinalIgnoreCase) + .Select(static group => group + .OrderBy(static reference => reference.Kind ?? string.Empty, StringComparer.Ordinal) + .ThenBy(static reference => reference.SourceTag ?? string.Empty, StringComparer.Ordinal) + .ThenBy(static reference => reference.Url, StringComparer.OrdinalIgnoreCase) + .First()) + .OrderBy(static reference => reference.Kind ?? string.Empty, StringComparer.Ordinal) + .ThenBy(static reference => reference.Url, StringComparer.OrdinalIgnoreCase); + } + + private static void TryAddReference( + ICollection references, + string? url, + string kind, + string? sourceTag, + string? summary, + AdvisoryProvenance provenance) + { + if (string.IsNullOrWhiteSpace(url)) + { + return; + } + + var candidate = url.Trim(); + if (!Uri.TryCreate(candidate, UriKind.Absolute, out var parsed)) + { + return; + } + + if (parsed.Scheme != Uri.UriSchemeHttp && parsed.Scheme != Uri.UriSchemeHttps) + { + return; + } + + var normalized = parsed.ToString(); + + try + { + references.Add(new AdvisoryReference(normalized, kind, sourceTag, summary, provenance)); + } + catch (ArgumentException) + { + // ignore invalid references + } + } + + private static bool TryFindEmbeddedUrl(string text, out string? url) + { + url = null; + if (string.IsNullOrWhiteSpace(text)) + { + return false; + } + + var tokens = text.Split(new[] { ' ', '\r', '\n', '\t' }, StringSplitOptions.RemoveEmptyEntries); + foreach (var token in tokens) + { + var trimmed = token.Trim().TrimEnd('.', ',', ')', ';', ']', '}'); + if (trimmed.Length == 0) + { + continue; + } + + if (!Uri.TryCreate(trimmed, UriKind.Absolute, out var parsed)) + { + continue; + } + + if (parsed.Scheme != Uri.UriSchemeHttp && parsed.Scheme != Uri.UriSchemeHttps) + { + continue; + } + + url = parsed.ToString(); + return true; + } + + return false; + } + + private static IEnumerable BuildAffectedPackages( + CertCcNoteDto dto, + CertCcNoteMetadata metadata, + string sourceName, + DateTimeOffset recordedAt) + { + var vendors = dto.Vendors ?? Array.Empty(); + var statuses = dto.VendorStatuses ?? Array.Empty(); + + if (vendors.Count == 0 && statuses.Count == 0) + { + return Array.Empty(); + } + + var statusLookup = statuses + .GroupBy(static status => NormalizeVendorKey(status.Vendor)) + .ToDictionary(static group => group.Key, static group => group.ToArray(), StringComparer.OrdinalIgnoreCase); + + var packages = new List(); + + foreach (var vendor in vendors.OrderBy(static v => v.Vendor, StringComparer.OrdinalIgnoreCase)) + { + var key = NormalizeVendorKey(vendor.Vendor); + var vendorStatuses = statusLookup.TryGetValue(key, out var value) + ? value + : Array.Empty(); + + if (BuildVendorPackage(vendor, vendorStatuses, sourceName, recordedAt) is { } package) + { + packages.Add(package); + } + + statusLookup.Remove(key); + } + + foreach (var remaining in statusLookup.Values) + { + if (remaining.Length == 0) + { + continue; + } + + var vendorName = remaining[0].Vendor; + var fallbackVendor = new CertCcVendorDto( + vendorName, + ContactDate: null, + StatementDate: null, + Updated: remaining + .Select(static status => status.DateUpdated) + .Where(static update => update.HasValue) + .OrderByDescending(static update => update) + .FirstOrDefault(), + Statement: remaining + .Select(static status => status.Statement) + .FirstOrDefault(static statement => !string.IsNullOrWhiteSpace(statement)), + Addendum: null, + References: remaining + .SelectMany(static status => status.References ?? Array.Empty()) + .ToArray()); + + if (BuildVendorPackage(fallbackVendor, remaining, sourceName, recordedAt) is { } package) + { + packages.Add(package); + } + } + + return packages + .OrderBy(static package => package.Identifier, StringComparer.OrdinalIgnoreCase) + .ToArray(); + } + + private static AffectedPackage? BuildVendorPackage( + CertCcVendorDto vendor, + IReadOnlyList statuses, + string sourceName, + DateTimeOffset recordedAt) + { + var vendorName = string.IsNullOrWhiteSpace(vendor.Vendor) + ? (statuses.FirstOrDefault()?.Vendor?.Trim() ?? string.Empty) + : vendor.Vendor.Trim(); + + if (vendorName.Length == 0) + { + return null; + } + + var packageProvenance = new AdvisoryProvenance(sourceName, "vendor", vendorName, recordedAt); + var rangeProvenance = new AdvisoryProvenance(sourceName, "vendor-range", vendorName, recordedAt); + + var patches = CertCcVendorStatementParser.Parse(vendor.Statement ?? string.Empty); + var normalizedVersions = BuildNormalizedVersions(vendorName, patches); + var vendorStatuses = BuildStatuses(vendorName, statuses, sourceName, recordedAt); + var primitives = BuildRangePrimitives(vendor, vendorStatuses, patches); + + var range = new AffectedVersionRange( + rangeKind: "vendor", + introducedVersion: null, + fixedVersion: null, + lastAffectedVersion: null, + rangeExpression: null, + provenance: rangeProvenance, + primitives: primitives); + + return new AffectedPackage( + AffectedPackageTypes.Vendor, + vendorName, + platform: null, + versionRanges: new[] { range }, + normalizedVersions: normalizedVersions, + statuses: vendorStatuses, + provenance: new[] { packageProvenance }); + } + + private static IReadOnlyList BuildNormalizedVersions( + string vendorName, + IReadOnlyList patches) + { + if (patches.Count == 0) + { + return Array.Empty(); + } + + var rules = new List(); + var seen = new HashSet(StringComparer.OrdinalIgnoreCase); + + foreach (var patch in patches) + { + if (string.IsNullOrWhiteSpace(patch.Version)) + { + continue; + } + + var version = patch.Version.Trim(); + if (!seen.Add($"{patch.Product}|{version}")) + { + continue; + } + + var notes = string.IsNullOrWhiteSpace(patch.Product) + ? vendorName + : $"{vendorName}::{patch.Product.Trim()}"; + + rules.Add(new NormalizedVersionRule( + VendorNormalizedVersionScheme, + NormalizedVersionRuleTypes.Exact, + value: version, + notes: notes)); + } + + return rules.Count == 0 ? Array.Empty() : rules; + } + + private static IReadOnlyList BuildStatuses( + string vendorName, + IReadOnlyList statuses, + string sourceName, + DateTimeOffset recordedAt) + { + if (statuses.Count == 0) + { + return Array.Empty(); + } + + var result = new List(); + var seen = new HashSet(StringComparer.OrdinalIgnoreCase); + + foreach (var status in statuses) + { + if (!AffectedPackageStatusCatalog.TryNormalize(status.Status, out var normalized)) + { + continue; + } + + var cve = status.CveId?.Trim() ?? string.Empty; + var key = string.IsNullOrWhiteSpace(cve) + ? normalized + : $"{normalized}|{cve}"; + + if (!seen.Add(key)) + { + continue; + } + + var provenance = new AdvisoryProvenance( + sourceName, + "vendor-status", + string.IsNullOrWhiteSpace(cve) ? vendorName : $"{vendorName}:{cve}", + recordedAt); + + result.Add(new AffectedPackageStatus(normalized, provenance)); + } + + return result + .OrderBy(static status => status.Status, StringComparer.Ordinal) + .ThenBy(static status => status.Provenance.Value ?? string.Empty, StringComparer.Ordinal) + .ToArray(); + } + + private static RangePrimitives? BuildRangePrimitives( + CertCcVendorDto vendor, + IReadOnlyList statuses, + IReadOnlyList patches) + { + var extensions = new Dictionary(StringComparer.OrdinalIgnoreCase); + + AddVendorExtension(extensions, "certcc.vendor.name", vendor.Vendor); + AddVendorExtension(extensions, "certcc.vendor.statement.raw", HtmlToPlainText(vendor.Statement ?? string.Empty), 2048); + AddVendorExtension(extensions, "certcc.vendor.addendum", HtmlToPlainText(vendor.Addendum ?? string.Empty), 1024); + AddVendorExtension(extensions, "certcc.vendor.contactDate", FormatDate(vendor.ContactDate)); + AddVendorExtension(extensions, "certcc.vendor.statementDate", FormatDate(vendor.StatementDate)); + AddVendorExtension(extensions, "certcc.vendor.updated", FormatDate(vendor.Updated)); + + if (vendor.References is { Count: > 0 }) + { + AddVendorExtension(extensions, "certcc.vendor.references", string.Join(" ", vendor.References)); + } + + if (statuses.Count > 0) + { + var serialized = string.Join(";", statuses + .Select(static status => status.Provenance.Value is { Length: > 0 } + ? $"{status.Provenance.Value.Split(':').Last()}={status.Status}" + : status.Status)); + + AddVendorExtension(extensions, "certcc.vendor.statuses", serialized); + } + + if (patches.Count > 0) + { + var serialized = string.Join(";", patches.Select(static patch => + { + var product = string.IsNullOrWhiteSpace(patch.Product) ? "unknown" : patch.Product.Trim(); + return $"{product}={patch.Version.Trim()}"; + })); + + AddVendorExtension(extensions, "certcc.vendor.patches", serialized, 2048); + } + + return extensions.Count == 0 + ? null + : new RangePrimitives(null, null, null, extensions); + } + + private static void AddVendorExtension(IDictionary extensions, string key, string? value, int maxLength = 512) + { + if (string.IsNullOrWhiteSpace(value)) + { + return; + } + + var trimmed = value.Trim(); + if (trimmed.Length > maxLength) + { + trimmed = trimmed[..maxLength].Trim(); + } + + if (trimmed.Length == 0) + { + return; + } + + extensions[key] = trimmed; + } + + private static string? FormatDate(DateTimeOffset? value) + => value?.ToUniversalTime().ToString("O", CultureInfo.InvariantCulture); + + private static string NormalizeVendorKey(string? value) + => string.IsNullOrWhiteSpace(value) ? string.Empty : value.Trim().ToLowerInvariant(); +} diff --git a/src/StellaOps.Feedser.Source.CertCc/Internal/CertCcNoteDto.cs b/src/StellaOps.Feedser.Source.CertCc/Internal/CertCcNoteDto.cs new file mode 100644 index 00000000..1e8913cb --- /dev/null +++ b/src/StellaOps.Feedser.Source.CertCc/Internal/CertCcNoteDto.cs @@ -0,0 +1,97 @@ +using System; +using System.Collections.Generic; + +namespace StellaOps.Feedser.Source.CertCc.Internal; + +internal sealed record CertCcNoteDto( + CertCcNoteMetadata Metadata, + IReadOnlyList Vendors, + IReadOnlyList VendorStatuses, + IReadOnlyList Vulnerabilities) +{ + public static CertCcNoteDto Empty { get; } = new( + CertCcNoteMetadata.Empty, + Array.Empty(), + Array.Empty(), + Array.Empty()); +} + +internal sealed record CertCcNoteMetadata( + string? VuId, + string IdNumber, + string Title, + string? Overview, + string? Summary, + DateTimeOffset? Published, + DateTimeOffset? Updated, + DateTimeOffset? Created, + int? Revision, + IReadOnlyList CveIds, + IReadOnlyList PublicUrls, + string? PrimaryUrl) +{ + public static CertCcNoteMetadata Empty { get; } = new( + VuId: null, + IdNumber: string.Empty, + Title: string.Empty, + Overview: null, + Summary: null, + Published: null, + Updated: null, + Created: null, + Revision: null, + CveIds: Array.Empty(), + PublicUrls: Array.Empty(), + PrimaryUrl: null); +} + +internal sealed record CertCcVendorDto( + string Vendor, + DateTimeOffset? ContactDate, + DateTimeOffset? StatementDate, + DateTimeOffset? Updated, + string? Statement, + string? Addendum, + IReadOnlyList References) +{ + public static CertCcVendorDto Empty { get; } = new( + Vendor: string.Empty, + ContactDate: null, + StatementDate: null, + Updated: null, + Statement: null, + Addendum: null, + References: Array.Empty()); +} + +internal sealed record CertCcVendorStatusDto( + string Vendor, + string CveId, + string Status, + string? Statement, + IReadOnlyList References, + DateTimeOffset? DateAdded, + DateTimeOffset? DateUpdated) +{ + public static CertCcVendorStatusDto Empty { get; } = new( + Vendor: string.Empty, + CveId: string.Empty, + Status: string.Empty, + Statement: null, + References: Array.Empty(), + DateAdded: null, + DateUpdated: null); +} + +internal sealed record CertCcVulnerabilityDto( + string CveId, + string? Description, + DateTimeOffset? DateAdded, + DateTimeOffset? DateUpdated) +{ + public static CertCcVulnerabilityDto Empty { get; } = new( + CveId: string.Empty, + Description: null, + DateAdded: null, + DateUpdated: null); +} diff --git a/src/StellaOps.Feedser.Source.CertCc/Internal/CertCcNoteParser.cs b/src/StellaOps.Feedser.Source.CertCc/Internal/CertCcNoteParser.cs new file mode 100644 index 00000000..41d5212e --- /dev/null +++ b/src/StellaOps.Feedser.Source.CertCc/Internal/CertCcNoteParser.cs @@ -0,0 +1,539 @@ +using System; +using System.Buffers; +using System.Collections.Generic; +using System.Globalization; +using System.Linq; +using System.Net; +using System.Text; +using System.Text.Json; +using System.Text.RegularExpressions; +using Markdig; +using StellaOps.Feedser.Source.Common.Html; +using StellaOps.Feedser.Source.Common.Url; + +namespace StellaOps.Feedser.Source.CertCc.Internal; + +internal static class CertCcNoteParser +{ + private static readonly MarkdownPipeline MarkdownPipeline = new MarkdownPipelineBuilder() + .UseAdvancedExtensions() + .UseSoftlineBreakAsHardlineBreak() + .DisableHtml() + .Build(); + + private static readonly HtmlContentSanitizer HtmlSanitizer = new(); + private static readonly Regex HtmlTagRegex = new("<[^>]+>", RegexOptions.Compiled | RegexOptions.CultureInvariant); + + public static CertCcNoteDto Parse( + ReadOnlySpan noteJson, + ReadOnlySpan vendorsJson, + ReadOnlySpan vulnerabilitiesJson, + ReadOnlySpan vendorStatusesJson) + { + using var noteDocument = JsonDocument.Parse(noteJson.ToArray()); + var (metadata, detailUri) = ParseNoteMetadata(noteDocument.RootElement); + + using var vendorsDocument = JsonDocument.Parse(vendorsJson.ToArray()); + var vendors = ParseVendors(vendorsDocument.RootElement, detailUri); + + using var vulnerabilitiesDocument = JsonDocument.Parse(vulnerabilitiesJson.ToArray()); + var vulnerabilities = ParseVulnerabilities(vulnerabilitiesDocument.RootElement); + + using var statusesDocument = JsonDocument.Parse(vendorStatusesJson.ToArray()); + var statuses = ParseVendorStatuses(statusesDocument.RootElement); + + return new CertCcNoteDto(metadata, vendors, statuses, vulnerabilities); + } + + public static CertCcNoteDto ParseNote(ReadOnlySpan noteJson) + { + using var noteDocument = JsonDocument.Parse(noteJson.ToArray()); + var (metadata, _) = ParseNoteMetadata(noteDocument.RootElement); + return new CertCcNoteDto(metadata, Array.Empty(), Array.Empty(), Array.Empty()); + } + + private static (CertCcNoteMetadata Metadata, Uri DetailUri) ParseNoteMetadata(JsonElement root) + { + if (root.ValueKind != JsonValueKind.Object) + { + throw new JsonException("CERT/CC note payload must be a JSON object."); + } + + var vuId = GetString(root, "vuid"); + var idNumber = GetString(root, "idnumber") ?? throw new JsonException("CERT/CC note missing idnumber."); + var title = GetString(root, "name") ?? throw new JsonException("CERT/CC note missing name."); + var detailUri = BuildDetailUri(idNumber); + + var overview = NormalizeMarkdownToPlainText(root, "overview", detailUri); + var summary = NormalizeMarkdownToPlainText(root, "clean_desc", detailUri); + if (string.IsNullOrWhiteSpace(summary)) + { + summary = NormalizeMarkdownToPlainText(root, "impact", detailUri); + } + + var published = ParseDate(root, "publicdate") ?? ParseDate(root, "datefirstpublished"); + var updated = ParseDate(root, "dateupdated"); + var created = ParseDate(root, "datecreated"); + var revision = ParseInt(root, "revision"); + + var cveIds = ExtractCveIds(root, "cveids"); + var references = ExtractReferenceList(root, "public", detailUri); + + var metadata = new CertCcNoteMetadata( + VuId: string.IsNullOrWhiteSpace(vuId) ? null : vuId.Trim(), + IdNumber: idNumber.Trim(), + Title: title.Trim(), + Overview: overview, + Summary: summary, + Published: published?.ToUniversalTime(), + Updated: updated?.ToUniversalTime(), + Created: created?.ToUniversalTime(), + Revision: revision, + CveIds: cveIds, + PublicUrls: references, + PrimaryUrl: detailUri.ToString()); + + return (metadata, detailUri); + } + + private static IReadOnlyList ParseVendors(JsonElement root, Uri baseUri) + { + if (root.ValueKind != JsonValueKind.Array || root.GetArrayLength() == 0) + { + return Array.Empty(); + } + + var parsed = new List(root.GetArrayLength()); + foreach (var element in root.EnumerateArray()) + { + if (element.ValueKind != JsonValueKind.Object) + { + continue; + } + + var vendor = GetString(element, "vendor"); + if (string.IsNullOrWhiteSpace(vendor)) + { + continue; + } + + var statement = NormalizeFreeformText(GetString(element, "statement")); + var addendum = NormalizeFreeformText(GetString(element, "addendum")); + var references = ExtractReferenceStringList(GetString(element, "references"), baseUri); + + parsed.Add(new CertCcVendorDto( + vendor.Trim(), + ContactDate: ParseDate(element, "contact_date"), + StatementDate: ParseDate(element, "statement_date"), + Updated: ParseDate(element, "dateupdated"), + Statement: statement, + Addendum: addendum, + References: references)); + } + + if (parsed.Count == 0) + { + return Array.Empty(); + } + + return parsed + .OrderBy(static vendor => vendor.Vendor, StringComparer.OrdinalIgnoreCase) + .ToArray(); + } + + private static IReadOnlyList ParseVulnerabilities(JsonElement root) + { + if (root.ValueKind != JsonValueKind.Array || root.GetArrayLength() == 0) + { + return Array.Empty(); + } + + var parsed = new List(root.GetArrayLength()); + foreach (var element in root.EnumerateArray()) + { + if (element.ValueKind != JsonValueKind.Object) + { + continue; + } + + var cve = GetString(element, "cve"); + if (string.IsNullOrWhiteSpace(cve)) + { + continue; + } + + parsed.Add(new CertCcVulnerabilityDto( + NormalizeCve(cve), + Description: NormalizeFreeformText(GetString(element, "description")), + DateAdded: ParseDate(element, "date_added"), + DateUpdated: ParseDate(element, "dateupdated"))); + } + + if (parsed.Count == 0) + { + return Array.Empty(); + } + + return parsed + .OrderBy(static vuln => vuln.CveId, StringComparer.OrdinalIgnoreCase) + .ToArray(); + } + + private static IReadOnlyList ParseVendorStatuses(JsonElement root) + { + if (root.ValueKind != JsonValueKind.Array || root.GetArrayLength() == 0) + { + return Array.Empty(); + } + + var parsed = new List(root.GetArrayLength()); + foreach (var element in root.EnumerateArray()) + { + if (element.ValueKind != JsonValueKind.Object) + { + continue; + } + + var vendor = GetString(element, "vendor"); + var cve = GetString(element, "vul"); + var status = GetString(element, "status"); + if (string.IsNullOrWhiteSpace(vendor) || string.IsNullOrWhiteSpace(cve) || string.IsNullOrWhiteSpace(status)) + { + continue; + } + + var references = ExtractReferenceStringList(GetString(element, "references"), baseUri: null); + parsed.Add(new CertCcVendorStatusDto( + vendor.Trim(), + NormalizeCve(cve), + status.Trim(), + NormalizeFreeformText(GetString(element, "statement")), + references, + DateAdded: ParseDate(element, "date_added"), + DateUpdated: ParseDate(element, "dateupdated"))); + } + + if (parsed.Count == 0) + { + return Array.Empty(); + } + + return parsed + .OrderBy(static entry => entry.CveId, StringComparer.OrdinalIgnoreCase) + .ThenBy(static entry => entry.Vendor, StringComparer.OrdinalIgnoreCase) + .ToArray(); + } + + private static string? NormalizeMarkdownToPlainText(JsonElement element, string propertyName, Uri baseUri) + => NormalizeMarkdownToPlainText(GetString(element, propertyName), baseUri); + + private static string? NormalizeMarkdownToPlainText(string? markdown, Uri baseUri) + { + if (string.IsNullOrWhiteSpace(markdown)) + { + return null; + } + + var normalized = NormalizeLineEndings(markdown.Trim()); + if (normalized.Length == 0) + { + return null; + } + + var html = Markdig.Markdown.ToHtml(normalized, MarkdownPipeline); + if (string.IsNullOrWhiteSpace(html)) + { + return null; + } + + var sanitized = HtmlSanitizer.Sanitize(html, baseUri); + if (string.IsNullOrWhiteSpace(sanitized)) + { + return null; + } + + var plain = ConvertHtmlToPlainText(sanitized); + return string.IsNullOrWhiteSpace(plain) ? null : plain; + } + + private static string? NormalizeFreeformText(string? value) + { + if (string.IsNullOrWhiteSpace(value)) + { + return null; + } + + var normalized = NormalizeLineEndings(value).Trim(); + if (normalized.Length == 0) + { + return null; + } + + var lines = normalized + .Split('\n') + .Select(static line => line.TrimEnd()) + .ToArray(); + + return string.Join('\n', lines).Trim(); + } + + private static string ConvertHtmlToPlainText(string html) + { + if (string.IsNullOrWhiteSpace(html)) + { + return string.Empty; + } + + var decoded = WebUtility.HtmlDecode(html); + decoded = decoded + .Replace("
    ", "\n", StringComparison.OrdinalIgnoreCase) + .Replace("
    ", "\n", StringComparison.OrdinalIgnoreCase) + .Replace("
    ", "\n", StringComparison.OrdinalIgnoreCase); + + decoded = Regex.Replace(decoded, "

    ", "\n\n", RegexOptions.IgnoreCase | RegexOptions.CultureInvariant); + decoded = Regex.Replace(decoded, "", "\n", RegexOptions.IgnoreCase | RegexOptions.CultureInvariant); + decoded = Regex.Replace(decoded, "
  • ", "- ", RegexOptions.IgnoreCase | RegexOptions.CultureInvariant); + decoded = Regex.Replace(decoded, "
    • ", "\n", RegexOptions.IgnoreCase | RegexOptions.CultureInvariant); + decoded = Regex.Replace(decoded, "", "\n", RegexOptions.IgnoreCase | RegexOptions.CultureInvariant); + decoded = Regex.Replace(decoded, "", " \t", RegexOptions.IgnoreCase | RegexOptions.CultureInvariant); + + decoded = HtmlTagRegex.Replace(decoded, string.Empty); + decoded = NormalizeLineEndings(decoded); + + var lines = decoded + .Split('\n', StringSplitOptions.RemoveEmptyEntries) + .Select(static line => line.Trim()) + .ToArray(); + + return string.Join('\n', lines).Trim(); + } + + private static IReadOnlyList ExtractReferenceList(JsonElement element, string propertyName, Uri baseUri) + { + if (!element.TryGetProperty(propertyName, out var raw) || raw.ValueKind != JsonValueKind.Array || raw.GetArrayLength() == 0) + { + return Array.Empty(); + } + + var references = new HashSet(StringComparer.OrdinalIgnoreCase); + foreach (var candidate in raw.EnumerateArray()) + { + if (candidate.ValueKind != JsonValueKind.String) + { + continue; + } + + var text = candidate.GetString(); + if (UrlNormalizer.TryNormalize(text, baseUri, out var normalized, stripFragment: true, forceHttps: false) && normalized is not null) + { + references.Add(normalized.ToString()); + } + } + + if (references.Count == 0) + { + return Array.Empty(); + } + + return references + .OrderBy(static url => url, StringComparer.OrdinalIgnoreCase) + .ToArray(); + } + + private static IReadOnlyList ExtractReferenceStringList(string? value, Uri? baseUri) + { + if (string.IsNullOrWhiteSpace(value)) + { + return Array.Empty(); + } + + var buffer = ArrayPool.Shared.Rent(16); + try + { + var count = 0; + var span = value.AsSpan(); + var start = 0; + + for (var index = 0; index < span.Length; index++) + { + var ch = span[index]; + if (ch == '\r' || ch == '\n') + { + if (index > start) + { + AppendSegment(span, start, index - start, baseUri, buffer, ref count); + } + + if (ch == '\r' && index + 1 < span.Length && span[index + 1] == '\n') + { + index++; + } + + start = index + 1; + } + } + + if (start < span.Length) + { + AppendSegment(span, start, span.Length - start, baseUri, buffer, ref count); + } + + if (count == 0) + { + return Array.Empty(); + } + + return buffer.AsSpan(0, count) + .ToArray() + .Distinct(StringComparer.OrdinalIgnoreCase) + .OrderBy(static url => url, StringComparer.OrdinalIgnoreCase) + .ToArray(); + } + finally + { + ArrayPool.Shared.Return(buffer, clearArray: true); + } + } + + private static void AppendSegment(ReadOnlySpan span, int start, int length, Uri? baseUri, string[] buffer, ref int count) + { + var segment = span.Slice(start, length).ToString().Trim(); + if (segment.Length == 0) + { + return; + } + + if (!UrlNormalizer.TryNormalize(segment, baseUri, out var normalized, stripFragment: true, forceHttps: false) || normalized is null) + { + return; + } + + if (count >= buffer.Length) + { + return; + } + + buffer[count++] = normalized.ToString(); + } + + private static IReadOnlyList ExtractCveIds(JsonElement element, string propertyName) + { + if (!element.TryGetProperty(propertyName, out var raw) || raw.ValueKind != JsonValueKind.Array || raw.GetArrayLength() == 0) + { + return Array.Empty(); + } + + var values = new HashSet(StringComparer.OrdinalIgnoreCase); + foreach (var entry in raw.EnumerateArray()) + { + if (entry.ValueKind != JsonValueKind.String) + { + continue; + } + + var text = entry.GetString(); + if (string.IsNullOrWhiteSpace(text)) + { + continue; + } + + values.Add(NormalizeCve(text)); + } + + if (values.Count == 0) + { + return Array.Empty(); + } + + return values + .OrderBy(static id => id, StringComparer.OrdinalIgnoreCase) + .ToArray(); + } + + private static string NormalizeCve(string value) + { + var trimmed = value.Trim(); + if (!trimmed.StartsWith("CVE-", StringComparison.OrdinalIgnoreCase)) + { + trimmed = $"CVE-{trimmed}"; + } + + var builder = new StringBuilder(trimmed.Length); + foreach (var ch in trimmed) + { + builder.Append(char.ToUpperInvariant(ch)); + } + + return builder.ToString(); + } + + private static string? GetString(JsonElement element, string propertyName) + { + if (element.ValueKind != JsonValueKind.Object) + { + return null; + } + + if (!element.TryGetProperty(propertyName, out var property)) + { + return null; + } + + return property.ValueKind switch + { + JsonValueKind.String => property.GetString(), + JsonValueKind.Number => property.ToString(), + _ => null, + }; + } + + private static DateTimeOffset? ParseDate(JsonElement element, string propertyName) + { + var text = GetString(element, propertyName); + if (string.IsNullOrWhiteSpace(text)) + { + return null; + } + + return DateTimeOffset.TryParse(text, CultureInfo.InvariantCulture, DateTimeStyles.AssumeUniversal, out var parsed) + ? parsed.ToUniversalTime() + : null; + } + + private static int? ParseInt(JsonElement element, string propertyName) + { + if (!element.TryGetProperty(propertyName, out var property)) + { + return null; + } + + if (property.ValueKind == JsonValueKind.Number && property.TryGetInt32(out var value)) + { + return value; + } + + var text = GetString(element, propertyName); + if (string.IsNullOrWhiteSpace(text)) + { + return null; + } + + return int.TryParse(text, NumberStyles.Integer, CultureInfo.InvariantCulture, out var parsed) ? parsed : (int?)null; + } + + private static Uri BuildDetailUri(string idNumber) + { + var sanitized = idNumber.Trim(); + return new Uri($"https://www.kb.cert.org/vuls/id/{sanitized}", UriKind.Absolute); + } + + private static string NormalizeLineEndings(string value) + { + if (value.IndexOf('\r') < 0) + { + return value; + } + + return value.Replace("\r\n", "\n", StringComparison.Ordinal).Replace('\r', '\n'); + } +} diff --git a/src/StellaOps.Feedser.Source.CertCc/Internal/CertCcSummaryPlan.cs b/src/StellaOps.Feedser.Source.CertCc/Internal/CertCcSummaryPlan.cs new file mode 100644 index 00000000..affaff82 --- /dev/null +++ b/src/StellaOps.Feedser.Source.CertCc/Internal/CertCcSummaryPlan.cs @@ -0,0 +1,22 @@ +using System; +using System.Collections.Generic; +using StellaOps.Feedser.Source.Common.Cursors; + +namespace StellaOps.Feedser.Source.CertCc.Internal; + +public sealed record CertCcSummaryPlan( + TimeWindow Window, + IReadOnlyList Requests, + TimeWindowCursorState NextState); + +public enum CertCcSummaryScope +{ + Monthly, + Yearly, +} + +public sealed record CertCcSummaryRequest( + Uri Uri, + CertCcSummaryScope Scope, + int Year, + int? Month); diff --git a/src/StellaOps.Feedser.Source.CertCc/Internal/CertCcSummaryPlanner.cs b/src/StellaOps.Feedser.Source.CertCc/Internal/CertCcSummaryPlanner.cs new file mode 100644 index 00000000..c7ba920e --- /dev/null +++ b/src/StellaOps.Feedser.Source.CertCc/Internal/CertCcSummaryPlanner.cs @@ -0,0 +1,96 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using Microsoft.Extensions.Options; +using StellaOps.Feedser.Source.CertCc.Configuration; +using StellaOps.Feedser.Source.Common.Cursors; + +namespace StellaOps.Feedser.Source.CertCc.Internal; + +/// +/// Computes which CERT/CC summary endpoints should be fetched for the next export window. +/// +public sealed class CertCcSummaryPlanner +{ + private readonly CertCcOptions _options; + private readonly TimeProvider _timeProvider; + + public CertCcSummaryPlanner( + IOptions options, + TimeProvider? timeProvider = null) + { + _options = (options ?? throw new ArgumentNullException(nameof(options))).Value ?? throw new ArgumentNullException(nameof(options)); + _options.Validate(); + _timeProvider = timeProvider ?? TimeProvider.System; + } + + public CertCcSummaryPlan CreatePlan(TimeWindowCursorState? state) + { + var now = _timeProvider.GetUtcNow(); + var window = TimeWindowCursorPlanner.GetNextWindow(now, state, _options.SummaryWindow); + var nextState = (state ?? TimeWindowCursorState.Empty).WithWindow(window); + + var months = EnumerateYearMonths(window.Start, window.End) + .Take(_options.MaxMonthlySummaries) + .ToArray(); + + if (months.Length == 0) + { + return new CertCcSummaryPlan(window, Array.Empty(), nextState); + } + + var requests = new List(months.Length * 2); + foreach (var month in months) + { + requests.Add(new CertCcSummaryRequest( + BuildMonthlyUri(month.Year, month.Month), + CertCcSummaryScope.Monthly, + month.Year, + month.Month)); + } + + foreach (var year in months.Select(static value => value.Year).Distinct().OrderBy(static year => year)) + { + requests.Add(new CertCcSummaryRequest( + BuildYearlyUri(year), + CertCcSummaryScope.Yearly, + year, + Month: null)); + } + + return new CertCcSummaryPlan(window, requests, nextState); + } + + private Uri BuildMonthlyUri(int year, int month) + { + var path = $"{year:D4}/{month:D2}/summary/"; + return new Uri(_options.BaseApiUri, path); + } + + private Uri BuildYearlyUri(int year) + { + var path = $"{year:D4}/summary/"; + return new Uri(_options.BaseApiUri, path); + } + + private static IEnumerable<(int Year, int Month)> EnumerateYearMonths(DateTimeOffset start, DateTimeOffset end) + { + if (end <= start) + { + yield break; + } + + var cursor = new DateTime(start.Year, start.Month, 1, 0, 0, 0, DateTimeKind.Utc); + var limit = new DateTime(end.Year, end.Month, 1, 0, 0, 0, DateTimeKind.Utc); + if (end.Day != 1 || end.TimeOfDay != TimeSpan.Zero) + { + limit = limit.AddMonths(1); + } + + while (cursor < limit) + { + yield return (cursor.Year, cursor.Month); + cursor = cursor.AddMonths(1); + } + } +} diff --git a/src/StellaOps.Feedser.Source.CertCc/Internal/CertCcVendorStatementParser.cs b/src/StellaOps.Feedser.Source.CertCc/Internal/CertCcVendorStatementParser.cs new file mode 100644 index 00000000..72bcdb15 --- /dev/null +++ b/src/StellaOps.Feedser.Source.CertCc/Internal/CertCcVendorStatementParser.cs @@ -0,0 +1,235 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text.RegularExpressions; + +namespace StellaOps.Feedser.Source.CertCc.Internal; + +internal static class CertCcVendorStatementParser +{ + private static readonly string[] PairSeparators = + { + "\t", + " - ", + " – ", + " — ", + " : ", + ": ", + " :", + ":", + }; + + private static readonly char[] BulletPrefixes = { '-', '*', '•', '+', '\t' }; + private static readonly char[] ProductDelimiters = { '/', ',', ';', '&' }; + + // Matches dotted numeric versions and simple alphanumeric suffixes (e.g., 4.4.3.6, 3.9.9.12, 10.2a) + private static readonly Regex VersionTokenRegex = new(@"(? Parse(string? statement) + { + if (string.IsNullOrWhiteSpace(statement)) + { + return Array.Empty(); + } + + var patches = new List(); + var lines = statement + .Replace("\r\n", "\n", StringComparison.Ordinal) + .Replace('\r', '\n') + .Split('\n', StringSplitOptions.RemoveEmptyEntries); + + foreach (var rawLine in lines) + { + var line = rawLine.Trim(); + if (line.Length == 0) + { + continue; + } + + line = TrimBulletPrefix(line); + if (line.Length == 0) + { + continue; + } + + if (!TrySplitLine(line, out var productSegment, out var versionSegment)) + { + continue; + } + + var versions = ExtractVersions(versionSegment); + if (versions.Count == 0) + { + continue; + } + + var products = ExtractProducts(productSegment); + if (products.Count == 0) + { + products.Add(string.Empty); + } + + if (versions.Count == products.Count) + { + for (var index = 0; index < versions.Count; index++) + { + patches.Add(new CertCcVendorPatch(products[index], versions[index], line)); + } + + continue; + } + + if (versions.Count > 1 && products.Count > versions.Count && products.Count % versions.Count == 0) + { + var groupSize = products.Count / versions.Count; + for (var versionIndex = 0; versionIndex < versions.Count; versionIndex++) + { + var start = versionIndex * groupSize; + var end = start + groupSize; + var version = versions[versionIndex]; + for (var productIndex = start; productIndex < end && productIndex < products.Count; productIndex++) + { + patches.Add(new CertCcVendorPatch(products[productIndex], version, line)); + } + } + + continue; + } + + var primaryVersion = versions[0]; + foreach (var product in products) + { + patches.Add(new CertCcVendorPatch(product, primaryVersion, line)); + } + } + + if (patches.Count == 0) + { + return Array.Empty(); + } + + return patches + .Where(static patch => !string.IsNullOrWhiteSpace(patch.Version)) + .Distinct(CertCcVendorPatch.Comparer) + .OrderBy(static patch => patch.Product, StringComparer.OrdinalIgnoreCase) + .ThenBy(static patch => patch.Version, StringComparer.Ordinal) + .ToArray(); + } + + private static string TrimBulletPrefix(string value) + { + var trimmed = value.TrimStart(BulletPrefixes).Trim(); + return trimmed.Length == 0 ? value.Trim() : trimmed; + } + + private static bool TrySplitLine(string line, out string productSegment, out string versionSegment) + { + foreach (var separator in PairSeparators) + { + var parts = line.Split(separator, 2, StringSplitOptions.TrimEntries); + if (parts.Length == 2) + { + productSegment = parts[0]; + versionSegment = parts[1]; + return true; + } + } + + var whitespaceSplit = line.Split(' ', StringSplitOptions.RemoveEmptyEntries); + if (whitespaceSplit.Length >= 2) + { + productSegment = string.Join(' ', whitespaceSplit[..^1]); + versionSegment = whitespaceSplit[^1]; + return true; + } + + productSegment = string.Empty; + versionSegment = string.Empty; + return false; + } + + private static List ExtractProducts(string segment) + { + if (string.IsNullOrWhiteSpace(segment)) + { + return new List(); + } + + var normalized = segment.Replace('\t', ' ').Trim(); + var tokens = normalized + .Split(ProductDelimiters, StringSplitOptions.RemoveEmptyEntries) + .Select(static token => token.Trim()) + .Where(static token => token.Length > 0) + .Distinct(StringComparer.OrdinalIgnoreCase) + .ToList(); + + return tokens; + } + + private static List ExtractVersions(string segment) + { + if (string.IsNullOrWhiteSpace(segment)) + { + return new List(); + } + + var matches = VersionTokenRegex.Matches(segment); + if (matches.Count == 0) + { + return new List(); + } + + var versions = new List(matches.Count); + foreach (Match match in matches) + { + if (match.Groups.Count == 0) + { + continue; + } + + var value = match.Groups[1].Value.Trim(); + if (value.Length == 0) + { + continue; + } + + versions.Add(value); + } + + return versions + .Distinct(StringComparer.OrdinalIgnoreCase) + .Take(32) + .ToList(); + } +} + +internal sealed record CertCcVendorPatch(string Product, string Version, string? RawLine) +{ + public static IEqualityComparer Comparer { get; } = new CertCcVendorPatchComparer(); + + private sealed class CertCcVendorPatchComparer : IEqualityComparer + { + public bool Equals(CertCcVendorPatch? x, CertCcVendorPatch? y) + { + if (ReferenceEquals(x, y)) + { + return true; + } + + if (x is null || y is null) + { + return false; + } + + return string.Equals(x.Product, y.Product, StringComparison.OrdinalIgnoreCase) + && string.Equals(x.Version, y.Version, StringComparison.OrdinalIgnoreCase); + } + + public int GetHashCode(CertCcVendorPatch obj) + { + var product = obj.Product?.ToLowerInvariant() ?? string.Empty; + var version = obj.Version?.ToLowerInvariant() ?? string.Empty; + return HashCode.Combine(product, version); + } + } +} diff --git a/src/StellaOps.Feedser.Source.CertCc/Jobs.cs b/src/StellaOps.Feedser.Source.CertCc/Jobs.cs new file mode 100644 index 00000000..eef41998 --- /dev/null +++ b/src/StellaOps.Feedser.Source.CertCc/Jobs.cs @@ -0,0 +1,22 @@ +using System; +using System.Threading; +using System.Threading.Tasks; +using StellaOps.Feedser.Core.Jobs; + +namespace StellaOps.Feedser.Source.CertCc; + +internal static class CertCcJobKinds +{ + public const string Fetch = "source:cert-cc:fetch"; +} + +internal sealed class CertCcFetchJob : IJob +{ + private readonly CertCcConnector _connector; + + public CertCcFetchJob(CertCcConnector connector) + => _connector = connector ?? throw new ArgumentNullException(nameof(connector)); + + public Task ExecuteAsync(JobExecutionContext context, CancellationToken cancellationToken) + => _connector.FetchAsync(context.Services, cancellationToken); +} diff --git a/src/StellaOps.Feedser.Source.CertCc/Properties/AssemblyInfo.cs b/src/StellaOps.Feedser.Source.CertCc/Properties/AssemblyInfo.cs new file mode 100644 index 00000000..96da4f07 --- /dev/null +++ b/src/StellaOps.Feedser.Source.CertCc/Properties/AssemblyInfo.cs @@ -0,0 +1,3 @@ +using System.Runtime.CompilerServices; + +[assembly: InternalsVisibleTo("StellaOps.Feedser.Source.CertCc.Tests")] diff --git a/src/StellaOps.Feedser.Source.CertCc/README.md b/src/StellaOps.Feedser.Source.CertCc/README.md new file mode 100644 index 00000000..a4bace30 --- /dev/null +++ b/src/StellaOps.Feedser.Source.CertCc/README.md @@ -0,0 +1,38 @@ +# CERT/CC Vulnerability Notes – Source Research + +## Canonical publication endpoints + +- **Public portal** – `https://www.kb.cert.org/vuls/` lists recently published Vulnerability Notes and exposes a “Subscribe to our feed” link for automation entry points.citeturn0search0 +- **Atom feed** – `https://www.kb.cert.org/vulfeed` returns an Atom 1.0 feed of the same notes (``, `<updated>`, `<summary>` HTML payload). Feed metadata advertises `rel="self"` at `https://kb.cert.org/vuls/atomfeed/`. Use conditional GET headers (`If-Modified-Since`, `If-None-Match`) to avoid refetching unchanged entries.citeturn0search2 + +## VINCE Vulnerability Note API + +The VINCE documentation describes an unauthenticated REST-style API for structured retrieval:citeturn1view0 + +| Endpoint | Payload | Notes | +| --- | --- | --- | +| `GET /vuls/api/{id}/` | Canonical note metadata (title, overview, markdown segments, timestamps, aliases). | Use numeric ID (e.g., `257161`). | +| `GET /vuls/api/{id}/vuls/` | Per-CVE vulnerability records tied to the note. | Includes CVE, description, timestamps. | +| `GET /vuls/api/{id}/vendors/` | Vendor statements per advisory. | Provides status text and optional references. | +| `GET /vuls/api/{id}/vendors/vuls/` | Vendor × vulnerability status matrix. | “known_affected” vs “known_not_affected” semantics. | +| `GET /vuls/api/vuls/cve/{cve}/` | Reverse lookup by CVE. | Returns combined note + vendor context. | +| `GET /vuls/api/{year}/summary/` | Annual summary listing (`count`, `notes[]`). | Year-month variants exist (`/{year}/{month}/summary/`). | +| `GET /vuls/api/{id}/csaf/` | CSAF 2.0 export generated by VINCE. | Useful for downstream CSAF tooling. | + +Operational considerations: + +- API responses are JSON (UTF-8) and publicly accessible; no authentication tokens or cookies are required.citeturn1view0 +- Monthly and annual summary endpoints enable incremental crawling without diffing the Atom feed. +- Expect high-volume notes to expose dozens of vendor records—prepare batching and pagination at the connector layer even though the API returns full arrays today. +- Apply polite backoff: the documentation does not publish explicit rate limits, but the kb.cert.org infrastructure throttles bursts; mirror existing backoff strategy (exponential with jitter) used by other connectors. + +## Historical data sets + +CERT/CC publishes a Vulnerability Data Archive (JSON exports plus tooling) for deep history or backfills. The archive is hosted on the SEI site with mirrored GitHub repositories containing normalized JSON conversions.citeturn0search3turn0search4 + +## Next steps for the connector + +1. Implement Atom polling for quick detection, with VINCE API lookups for structured details. `CertCcSummaryPlanner` already computes the VINCE year/month summary URIs to fetch per window; wire this into the fetch job and persist the resulting `TimeWindowCursorState`. +2. Persist `updated` timestamps and VINCE `revision` counters to drive resume logic. +3. Capture vendor statements/CSAF exports to populate range primitives once model hooks exist. +4. Evaluate using the data archive for seed fixtures covering legacy notes (pre-2010).*** diff --git a/src/StellaOps.Feedser.Source.CertCc/StellaOps.Feedser.Source.CertCc.csproj b/src/StellaOps.Feedser.Source.CertCc/StellaOps.Feedser.Source.CertCc.csproj new file mode 100644 index 00000000..04f9158c --- /dev/null +++ b/src/StellaOps.Feedser.Source.CertCc/StellaOps.Feedser.Source.CertCc.csproj @@ -0,0 +1,16 @@ +<Project Sdk="Microsoft.NET.Sdk"> + + <PropertyGroup> + <TargetFramework>net10.0</TargetFramework> + <ImplicitUsings>enable</ImplicitUsings> + <Nullable>enable</Nullable> + </PropertyGroup> + + <ItemGroup> + <ProjectReference Include="../StellaOps.Plugin/StellaOps.Plugin.csproj" /> + + <ProjectReference Include="../StellaOps.Feedser.Source.Common/StellaOps.Feedser.Source.Common.csproj" /> + <ProjectReference Include="../StellaOps.Feedser.Models/StellaOps.Feedser.Models.csproj" /> + <ProjectReference Include="../StellaOps.Feedser.Storage.Mongo/StellaOps.Feedser.Storage.Mongo.csproj" /> + </ItemGroup> +</Project> diff --git a/src/StellaOps.Feedser.Source.CertCc/TASKS.md b/src/StellaOps.Feedser.Source.CertCc/TASKS.md new file mode 100644 index 00000000..1ece5afb --- /dev/null +++ b/src/StellaOps.Feedser.Source.CertCc/TASKS.md @@ -0,0 +1,12 @@ +# TASKS +| Task | Owner(s) | Depends on | Notes | +|---|---|---|---| +|Document CERT/CC advisory sources|BE-Conn-CERTCC|Research|**DONE (2025-10-10)** – Catalogued Atom feed + VINCE API endpoints and archive references in `README.md`; include polling/backoff guidance.| +|Fetch pipeline & state tracking|BE-Conn-CERTCC|Source.Common, Storage.Mongo|**DOING (2025-10-10)** – Summary planner + fetch job now persist monthly/yearly VINCE JSON to `DocumentStore` while advancing a `TimeWindowCursorState`; follow-up: fan out summary payloads into per-note detail fetch queue and store hydrated cursor state.| +|VINCE note detail fetcher|BE-Conn-CERTCC|Source.Common, Storage.Mongo|**TODO** – Read summary documents, enqueue unique VU IDs, fetch `/vuls/api/{id}/`, `/vuls/api/{id}/vendors/`, `/vuls/api/{id}/vuls/`, persist raw JSON, dedupe via SHA/ETag, and record retry/backoff metadata.| +|DTO & parser implementation|BE-Conn-CERTCC|Source.Common|**TODO** – Model VINCE JSON (note + vendors + vulnerabilities), create strongly typed DTOs, normalise markdown into HTML-safe fragments, and surface vendor/product impact statements.| +|Canonical mapping & range primitives|BE-Conn-CERTCC|Models|**TODO** – Map VINCE DTOs to canonical advisories (aliases: VU#, CVE), vendors/products into RangePrimitives (`certcc.vendor`, affected platforms), and emit mitigation references.| +|Deterministic fixtures/tests|QA|Testing|**BLOCKED (awaiting SemVer + decisionReason rollout)** – Snapshot harness in place, but regeneration fails until upstream model/storage changes (SemVer primitives, provenance decision reasons) land in trunk.| +|Snapshot coverage handoff|QA|Models, Merge|**BLOCKED (2025-10-11)** – Fixture refresh cannot complete while `AdvisoryProvenance.decisionReason` and SemVer normalization packages remain absent from repo; re-run once Models/Storage merge sprint lands.| +|FEEDCONN-CERTCC-02-012 Schema sync & snapshot regen follow-up|QA, BE-Conn-CERTCC|Models `FEEDMODELS-SCHEMA-01-001`/`-002`/`-003`, Storage `FEEDSTORAGE-DATA-02-001`|TODO – After repo picks up the SemVer style and provenance changes, rerun connector snapshot tests with `UPDATE_CERTCC_FIXTURES=1`, publish regenerated fixtures, and deliver Merge handoff diff.| +|Telemetry & documentation|DevEx|Docs|**TODO** – Add logging/metrics, document connector behaviour, update backlog once implementation completes.| diff --git a/src/StellaOps.Feedser.Source.CertFr.Tests/CertFr/CertFrConnectorTests.cs b/src/StellaOps.Feedser.Source.CertFr.Tests/CertFr/CertFrConnectorTests.cs new file mode 100644 index 00000000..92ccb065 --- /dev/null +++ b/src/StellaOps.Feedser.Source.CertFr.Tests/CertFr/CertFrConnectorTests.cs @@ -0,0 +1,313 @@ +using System; +using System.IO; +using System.Linq; +using System.Net; +using System.Net.Http; +using System.Net.Http.Headers; +using System.Text; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Http; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Logging.Abstractions; +using Microsoft.Extensions.Options; +using Microsoft.Extensions.Time.Testing; +using MongoDB.Bson; +using MongoDB.Driver; +using StellaOps.Feedser.Source.CertFr; +using StellaOps.Feedser.Source.CertFr.Configuration; +using StellaOps.Feedser.Source.Common; +using StellaOps.Feedser.Source.Common.Http; +using StellaOps.Feedser.Source.Common.Testing; +using StellaOps.Feedser.Storage.Mongo; +using StellaOps.Feedser.Storage.Mongo.Advisories; +using StellaOps.Feedser.Storage.Mongo.Documents; +using StellaOps.Feedser.Storage.Mongo.Dtos; +using StellaOps.Feedser.Testing; +using StellaOps.Feedser.Models; + +namespace StellaOps.Feedser.Source.CertFr.Tests; + +[Collection("mongo-fixture")] +public sealed class CertFrConnectorTests : IAsyncLifetime +{ + private static readonly Uri FeedUri = new("https://www.cert.ssi.gouv.fr/feed/alertes/"); + private static readonly Uri FirstDetailUri = new("https://www.cert.ssi.gouv.fr/alerte/AV-2024.001/"); + private static readonly Uri SecondDetailUri = new("https://www.cert.ssi.gouv.fr/alerte/AV-2024.002/"); + + private readonly MongoIntegrationFixture _fixture; + private readonly FakeTimeProvider _timeProvider; + private readonly CannedHttpMessageHandler _handler; + + public CertFrConnectorTests(MongoIntegrationFixture fixture) + { + _fixture = fixture; + _timeProvider = new FakeTimeProvider(new DateTimeOffset(2024, 10, 3, 0, 0, 0, TimeSpan.Zero)); + _handler = new CannedHttpMessageHandler(); + } + + [Fact] + public async Task FetchParseMap_ProducesDeterministicSnapshot() + { + await using var provider = await BuildServiceProviderAsync(); + SeedFeed(); + SeedDetailResponses(); + + var connector = provider.GetRequiredService<CertFrConnector>(); + await connector.FetchAsync(provider, CancellationToken.None); + _timeProvider.Advance(TimeSpan.FromMinutes(1)); + await connector.ParseAsync(provider, CancellationToken.None); + await connector.MapAsync(provider, CancellationToken.None); + + var advisoryStore = provider.GetRequiredService<IAdvisoryStore>(); + var advisories = await advisoryStore.GetRecentAsync(10, CancellationToken.None); + Assert.Equal(2, advisories.Count); + + var snapshot = SnapshotSerializer.ToSnapshot(advisories.OrderBy(static a => a.AdvisoryKey, StringComparer.Ordinal).ToArray()); + var expected = ReadFixture("certfr-advisories.snapshot.json"); + var normalizedSnapshot = Normalize(snapshot); + var normalizedExpected = Normalize(expected); + if (!string.Equals(normalizedExpected, normalizedSnapshot, StringComparison.Ordinal)) + { + var actualPath = Path.Combine(AppContext.BaseDirectory, "Source", "CertFr", "Fixtures", "certfr-advisories.actual.json"); + Directory.CreateDirectory(Path.GetDirectoryName(actualPath)!); + File.WriteAllText(actualPath, snapshot); + } + + Assert.Equal(normalizedExpected, normalizedSnapshot); + + var documentStore = provider.GetRequiredService<IDocumentStore>(); + var firstDocument = await documentStore.FindBySourceAndUriAsync(CertFrConnectorPlugin.SourceName, FirstDetailUri.ToString(), CancellationToken.None); + Assert.NotNull(firstDocument); + Assert.Equal(DocumentStatuses.Mapped, firstDocument!.Status); + + var secondDocument = await documentStore.FindBySourceAndUriAsync(CertFrConnectorPlugin.SourceName, SecondDetailUri.ToString(), CancellationToken.None); + Assert.NotNull(secondDocument); + Assert.Equal(DocumentStatuses.Mapped, secondDocument!.Status); + + var stateRepository = provider.GetRequiredService<ISourceStateRepository>(); + var state = await stateRepository.TryGetAsync(CertFrConnectorPlugin.SourceName, CancellationToken.None); + Assert.NotNull(state); + Assert.True(state!.Cursor.TryGetValue("pendingDocuments", out var pendingDocs) && pendingDocs.AsBsonArray.Count == 0); + Assert.True(state.Cursor.TryGetValue("pendingMappings", out var pendingMaps) && pendingMaps.AsBsonArray.Count == 0); + } + + [Fact] + public async Task FetchFailure_RecordsBackoffAndReason() + { + await using var provider = await BuildServiceProviderAsync(); + _handler.AddResponse(FeedUri, () => new HttpResponseMessage(HttpStatusCode.InternalServerError) + { + Content = new StringContent("feed error", Encoding.UTF8, "text/plain"), + }); + + var connector = provider.GetRequiredService<CertFrConnector>(); + await Assert.ThrowsAsync<HttpRequestException>(() => connector.FetchAsync(provider, CancellationToken.None)); + + var stateRepository = provider.GetRequiredService<ISourceStateRepository>(); + var state = await stateRepository.TryGetAsync(CertFrConnectorPlugin.SourceName, CancellationToken.None); + Assert.NotNull(state); + Assert.Equal(1, state!.FailCount); + Assert.NotNull(state.LastFailureReason); + Assert.Contains("500", state.LastFailureReason, StringComparison.Ordinal); + Assert.NotNull(state.BackoffUntil); + Assert.True(state.BackoffUntil > _timeProvider.GetUtcNow()); + } + + [Fact] + public async Task Fetch_NotModifiedResponsesMaintainDocumentState() + { + await using var provider = await BuildServiceProviderAsync(); + SeedFeed(); + SeedDetailResponses(); + + var connector = provider.GetRequiredService<CertFrConnector>(); + await connector.FetchAsync(provider, CancellationToken.None); + await connector.ParseAsync(provider, CancellationToken.None); + await connector.MapAsync(provider, CancellationToken.None); + + var documentStore = provider.GetRequiredService<IDocumentStore>(); + var firstDocument = await documentStore.FindBySourceAndUriAsync(CertFrConnectorPlugin.SourceName, FirstDetailUri.ToString(), CancellationToken.None); + Assert.NotNull(firstDocument); + Assert.Equal(DocumentStatuses.Mapped, firstDocument!.Status); + + var secondDocument = await documentStore.FindBySourceAndUriAsync(CertFrConnectorPlugin.SourceName, SecondDetailUri.ToString(), CancellationToken.None); + Assert.NotNull(secondDocument); + Assert.Equal(DocumentStatuses.Mapped, secondDocument!.Status); + + SeedFeed(); + SeedNotModifiedDetailResponses(); + + await connector.FetchAsync(provider, CancellationToken.None); + + firstDocument = await documentStore.FindBySourceAndUriAsync(CertFrConnectorPlugin.SourceName, FirstDetailUri.ToString(), CancellationToken.None); + Assert.NotNull(firstDocument); + Assert.Equal(DocumentStatuses.Mapped, firstDocument!.Status); + + secondDocument = await documentStore.FindBySourceAndUriAsync(CertFrConnectorPlugin.SourceName, SecondDetailUri.ToString(), CancellationToken.None); + Assert.NotNull(secondDocument); + Assert.Equal(DocumentStatuses.Mapped, secondDocument!.Status); + + var stateRepository = provider.GetRequiredService<ISourceStateRepository>(); + var state = await stateRepository.TryGetAsync(CertFrConnectorPlugin.SourceName, CancellationToken.None); + Assert.NotNull(state); + Assert.True(state!.Cursor.TryGetValue("pendingDocuments", out var pendingDocs) && pendingDocs.AsBsonArray.Count == 0); + Assert.True(state.Cursor.TryGetValue("pendingMappings", out var pendingMaps) && pendingMaps.AsBsonArray.Count == 0); + } + + [Fact] + public async Task Fetch_DuplicateContentSkipsRequeue() + { + await using var provider = await BuildServiceProviderAsync(); + SeedFeed(); + SeedDetailResponses(); + + var connector = provider.GetRequiredService<CertFrConnector>(); + await connector.FetchAsync(provider, CancellationToken.None); + await connector.ParseAsync(provider, CancellationToken.None); + await connector.MapAsync(provider, CancellationToken.None); + + var documentStore = provider.GetRequiredService<IDocumentStore>(); + var firstDocument = await documentStore.FindBySourceAndUriAsync(CertFrConnectorPlugin.SourceName, FirstDetailUri.ToString(), CancellationToken.None); + Assert.NotNull(firstDocument); + Assert.Equal(DocumentStatuses.Mapped, firstDocument!.Status); + + var secondDocument = await documentStore.FindBySourceAndUriAsync(CertFrConnectorPlugin.SourceName, SecondDetailUri.ToString(), CancellationToken.None); + Assert.NotNull(secondDocument); + Assert.Equal(DocumentStatuses.Mapped, secondDocument!.Status); + + SeedFeed(); + SeedDetailResponses(); + + await connector.FetchAsync(provider, CancellationToken.None); + await connector.ParseAsync(provider, CancellationToken.None); + await connector.MapAsync(provider, CancellationToken.None); + + firstDocument = await documentStore.FindBySourceAndUriAsync(CertFrConnectorPlugin.SourceName, FirstDetailUri.ToString(), CancellationToken.None); + Assert.NotNull(firstDocument); + Assert.Equal(DocumentStatuses.Mapped, firstDocument!.Status); + + secondDocument = await documentStore.FindBySourceAndUriAsync(CertFrConnectorPlugin.SourceName, SecondDetailUri.ToString(), CancellationToken.None); + Assert.NotNull(secondDocument); + Assert.Equal(DocumentStatuses.Mapped, secondDocument!.Status); + + var stateRepository = provider.GetRequiredService<ISourceStateRepository>(); + var state = await stateRepository.TryGetAsync(CertFrConnectorPlugin.SourceName, CancellationToken.None); + Assert.NotNull(state); + Assert.True(state!.Cursor.TryGetValue("pendingDocuments", out var pendingDocs) && pendingDocs.AsBsonArray.Count == 0); + Assert.True(state.Cursor.TryGetValue("pendingMappings", out var pendingMaps) && pendingMaps.AsBsonArray.Count == 0); + } + + private async Task<ServiceProvider> BuildServiceProviderAsync() + { + await _fixture.Client.DropDatabaseAsync(_fixture.Database.DatabaseNamespace.DatabaseName); + _handler.Clear(); + + var services = new ServiceCollection(); + services.AddLogging(builder => builder.AddProvider(NullLoggerProvider.Instance)); + services.AddSingleton<TimeProvider>(_timeProvider); + services.AddSingleton(_handler); + + services.AddMongoStorage(options => + { + options.ConnectionString = _fixture.Runner.ConnectionString; + options.DatabaseName = _fixture.Database.DatabaseNamespace.DatabaseName; + options.CommandTimeout = TimeSpan.FromSeconds(5); + }); + + services.AddSourceCommon(); + services.AddCertFrConnector(opts => + { + opts.FeedUri = FeedUri; + opts.InitialBackfill = TimeSpan.FromDays(30); + opts.WindowOverlap = TimeSpan.FromDays(2); + opts.MaxItemsPerFetch = 50; + }); + + services.Configure<HttpClientFactoryOptions>(CertFrOptions.HttpClientName, builderOptions => + { + builderOptions.HttpMessageHandlerBuilderActions.Add(builder => + { + builder.PrimaryHandler = _handler; + }); + }); + + var provider = services.BuildServiceProvider(); + var bootstrapper = provider.GetRequiredService<MongoBootstrapper>(); + await bootstrapper.InitializeAsync(CancellationToken.None); + return provider; + } + + private void SeedFeed() + { + _handler.AddTextResponse(FeedUri, ReadFixture("certfr-feed.xml"), "application/atom+xml"); + } + + private void SeedDetailResponses() + { + AddDetailResponse(FirstDetailUri, "certfr-detail-AV-2024-001.html", "\"certfr-001\""); + AddDetailResponse(SecondDetailUri, "certfr-detail-AV-2024-002.html", "\"certfr-002\""); + } + + private void SeedNotModifiedDetailResponses() + { + AddNotModifiedResponse(FirstDetailUri, "\"certfr-001\""); + AddNotModifiedResponse(SecondDetailUri, "\"certfr-002\""); + } + + private void AddDetailResponse(Uri uri, string fixture, string? etag) + { + _handler.AddResponse(uri, () => + { + var response = new HttpResponseMessage(HttpStatusCode.OK) + { + Content = new StringContent(ReadFixture(fixture), Encoding.UTF8, "text/html"), + }; + + if (!string.IsNullOrEmpty(etag)) + { + response.Headers.ETag = new EntityTagHeaderValue(etag); + } + + return response; + }); + } + + private void AddNotModifiedResponse(Uri uri, string? etag) + { + _handler.AddResponse(uri, () => + { + var response = new HttpResponseMessage(HttpStatusCode.NotModified); + if (!string.IsNullOrEmpty(etag)) + { + response.Headers.ETag = new EntityTagHeaderValue(etag); + } + + return response; + }); + } + + private static string ReadFixture(string filename) + { + var baseDirectory = AppContext.BaseDirectory; + var primary = Path.Combine(baseDirectory, "Source", "CertFr", "Fixtures", filename); + if (File.Exists(primary)) + { + return File.ReadAllText(primary); + } + + var fallback = Path.Combine(baseDirectory, "CertFr", "Fixtures", filename); + return File.ReadAllText(fallback); + } + + private static string Normalize(string value) + => value.Replace("\r\n", "\n", StringComparison.Ordinal); + + public Task InitializeAsync() => Task.CompletedTask; + + public async Task DisposeAsync() + { + await _fixture.Client.DropDatabaseAsync(_fixture.Database.DatabaseNamespace.DatabaseName); + } +} diff --git a/src/StellaOps.Feedser.Source.CertFr.Tests/CertFr/Fixtures/certfr-advisories.snapshot.json b/src/StellaOps.Feedser.Source.CertFr.Tests/CertFr/Fixtures/certfr-advisories.snapshot.json new file mode 100644 index 00000000..d0340a94 --- /dev/null +++ b/src/StellaOps.Feedser.Source.CertFr.Tests/CertFr/Fixtures/certfr-advisories.snapshot.json @@ -0,0 +1,205 @@ +[ + { + "advisoryKey": "cert-fr/AV-2024.001", + "affectedPackages": [ + { + "identifier": "AV-2024.001", + "platform": null, + "provenance": [ + { + "fieldMask": [], + "kind": "document", + "recordedAt": "2024-10-03T00:01:00+00:00", + "source": "cert-fr", + "value": "https://www.cert.ssi.gouv.fr/alerte/AV-2024.001/" + } + ], + "statuses": [], + "type": "vendor", + "versionRanges": [ + { + "fixedVersion": null, + "introducedVersion": null, + "lastAffectedVersion": null, + "primitives": { + "evr": null, + "hasVendorExtensions": true, + "nevra": null, + "semVer": null, + "vendorExtensions": { + "certfr.summary": "Résumé de la première alerte.", + "certfr.content": "AV-2024.001 Alerte CERT-FR AV-2024.001 L'exploitation active de la vulnérabilité est surveillée. Consultez les indications du fournisseur .", + "certfr.reference.count": "1" + } + }, + "provenance": { + "fieldMask": [], + "kind": "document", + "recordedAt": "2024-10-03T00:01:00+00:00", + "source": "cert-fr", + "value": "https://www.cert.ssi.gouv.fr/alerte/AV-2024.001/" + }, + "rangeExpression": null, + "rangeKind": "vendor" + } + ] + } + ], + "aliases": [ + "CERT-FR:AV-2024.001" + ], + "cvssMetrics": [], + "exploitKnown": false, + "language": "fr", + "modified": null, + "provenance": [ + { + "fieldMask": [], + "kind": "document", + "recordedAt": "2024-10-03T00:01:00+00:00", + "source": "cert-fr", + "value": "https://www.cert.ssi.gouv.fr/alerte/AV-2024.001/" + } + ], + "published": "2024-10-03T00:00:00+00:00", + "references": [ + { + "kind": "reference", + "provenance": { + "fieldMask": [], + "kind": "document", + "recordedAt": "2024-10-03T00:01:00+00:00", + "source": "cert-fr", + "value": "https://www.cert.ssi.gouv.fr/alerte/AV-2024.001/" + }, + "sourceTag": null, + "summary": null, + "url": "https://vendor.example.com/patch" + }, + { + "kind": "advisory", + "provenance": { + "fieldMask": [], + "kind": "document", + "recordedAt": "2024-10-03T00:01:00+00:00", + "source": "cert-fr", + "value": "https://www.cert.ssi.gouv.fr/alerte/AV-2024.001/" + }, + "sourceTag": "cert-fr", + "summary": "Résumé de la première alerte.", + "url": "https://www.cert.ssi.gouv.fr/alerte/AV-2024.001/" + } + ], + "severity": null, + "summary": "Résumé de la première alerte.", + "title": "AV-2024.001 - Première alerte" + }, + { + "advisoryKey": "cert-fr/AV-2024.002", + "affectedPackages": [ + { + "identifier": "AV-2024.002", + "platform": null, + "provenance": [ + { + "fieldMask": [], + "kind": "document", + "recordedAt": "2024-10-03T00:01:00+00:00", + "source": "cert-fr", + "value": "https://www.cert.ssi.gouv.fr/alerte/AV-2024.002/" + } + ], + "statuses": [], + "type": "vendor", + "versionRanges": [ + { + "fixedVersion": null, + "introducedVersion": null, + "lastAffectedVersion": null, + "primitives": { + "evr": null, + "hasVendorExtensions": true, + "nevra": null, + "semVer": null, + "vendorExtensions": { + "certfr.summary": "Résumé de la deuxième alerte.", + "certfr.content": "AV-2024.002 Alerte CERT-FR AV-2024.002 Des correctifs sont disponibles pour plusieurs produits. Note de mise à jour Correctif", + "certfr.reference.count": "2" + } + }, + "provenance": { + "fieldMask": [], + "kind": "document", + "recordedAt": "2024-10-03T00:01:00+00:00", + "source": "cert-fr", + "value": "https://www.cert.ssi.gouv.fr/alerte/AV-2024.002/" + }, + "rangeExpression": null, + "rangeKind": "vendor" + } + ] + } + ], + "aliases": [ + "CERT-FR:AV-2024.002" + ], + "cvssMetrics": [], + "exploitKnown": false, + "language": "fr", + "modified": null, + "provenance": [ + { + "fieldMask": [], + "kind": "document", + "recordedAt": "2024-10-03T00:01:00+00:00", + "source": "cert-fr", + "value": "https://www.cert.ssi.gouv.fr/alerte/AV-2024.002/" + } + ], + "published": "2024-10-03T00:00:00+00:00", + "references": [ + { + "kind": "reference", + "provenance": { + "fieldMask": [], + "kind": "document", + "recordedAt": "2024-10-03T00:01:00+00:00", + "source": "cert-fr", + "value": "https://www.cert.ssi.gouv.fr/alerte/AV-2024.002/" + }, + "sourceTag": null, + "summary": null, + "url": "https://support.example.com/kb/KB-1234" + }, + { + "kind": "reference", + "provenance": { + "fieldMask": [], + "kind": "document", + "recordedAt": "2024-10-03T00:01:00+00:00", + "source": "cert-fr", + "value": "https://www.cert.ssi.gouv.fr/alerte/AV-2024.002/" + }, + "sourceTag": null, + "summary": null, + "url": "https://support.example.com/kb/KB-5678" + }, + { + "kind": "advisory", + "provenance": { + "fieldMask": [], + "kind": "document", + "recordedAt": "2024-10-03T00:01:00+00:00", + "source": "cert-fr", + "value": "https://www.cert.ssi.gouv.fr/alerte/AV-2024.002/" + }, + "sourceTag": "cert-fr", + "summary": "Résumé de la deuxième alerte.", + "url": "https://www.cert.ssi.gouv.fr/alerte/AV-2024.002/" + } + ], + "severity": null, + "summary": "Résumé de la deuxième alerte.", + "title": "AV-2024.002 - Deuxième alerte" + } +] \ No newline at end of file diff --git a/src/StellaOps.Feedser.Source.CertFr.Tests/CertFr/Fixtures/certfr-detail-AV-2024-001.html b/src/StellaOps.Feedser.Source.CertFr.Tests/CertFr/Fixtures/certfr-detail-AV-2024-001.html new file mode 100644 index 00000000..29d5afec --- /dev/null +++ b/src/StellaOps.Feedser.Source.CertFr.Tests/CertFr/Fixtures/certfr-detail-AV-2024-001.html @@ -0,0 +1,8 @@ +<html> + <head><title>AV-2024.001 + +

    Alerte CERT-FR AV-2024.001

    +

    L'exploitation active de la vulnérabilité est surveillée.

    +

    Consultez les indications du fournisseur.

    + + diff --git a/src/StellaOps.Feedser.Source.CertFr.Tests/CertFr/Fixtures/certfr-detail-AV-2024-002.html b/src/StellaOps.Feedser.Source.CertFr.Tests/CertFr/Fixtures/certfr-detail-AV-2024-002.html new file mode 100644 index 00000000..52f72fd8 --- /dev/null +++ b/src/StellaOps.Feedser.Source.CertFr.Tests/CertFr/Fixtures/certfr-detail-AV-2024-002.html @@ -0,0 +1,11 @@ + + AV-2024.002 + +

    Alerte CERT-FR AV-2024.002

    +

    Des correctifs sont disponibles pour plusieurs produits.

    + + + diff --git a/src/StellaOps.Feedser.Source.CertFr.Tests/CertFr/Fixtures/certfr-feed.xml b/src/StellaOps.Feedser.Source.CertFr.Tests/CertFr/Fixtures/certfr-feed.xml new file mode 100644 index 00000000..904cafd3 --- /dev/null +++ b/src/StellaOps.Feedser.Source.CertFr.Tests/CertFr/Fixtures/certfr-feed.xml @@ -0,0 +1,22 @@ + + + + CERT-FR Alertes + https://www.cert.ssi.gouv.fr/ + Alertes example feed + + AV-2024.001 - Première alerte + https://www.cert.ssi.gouv.fr/alerte/AV-2024.001/ + + Thu, 03 Oct 2024 09:00:00 +0000 + AV-2024.001 + + + AV-2024.002 - Deuxième alerte + https://www.cert.ssi.gouv.fr/alerte/AV-2024.002/ + + Thu, 03 Oct 2024 11:30:00 +0000 + AV-2024.002 + + + diff --git a/src/StellaOps.Feedser.Source.CertFr.Tests/StellaOps.Feedser.Source.CertFr.Tests.csproj b/src/StellaOps.Feedser.Source.CertFr.Tests/StellaOps.Feedser.Source.CertFr.Tests.csproj new file mode 100644 index 00000000..6cf96d45 --- /dev/null +++ b/src/StellaOps.Feedser.Source.CertFr.Tests/StellaOps.Feedser.Source.CertFr.Tests.csproj @@ -0,0 +1,16 @@ + + + net10.0 + enable + enable + + + + + + + + + + + diff --git a/src/StellaOps.Feedser.Source.CertFr/AGENTS.md b/src/StellaOps.Feedser.Source.CertFr/AGENTS.md new file mode 100644 index 00000000..5e5cf3e2 --- /dev/null +++ b/src/StellaOps.Feedser.Source.CertFr/AGENTS.md @@ -0,0 +1,27 @@ +# AGENTS +## Role +ANSSI CERT-FR advisories connector (avis/alertes) providing national enrichment: advisory metadata, CVE links, mitigation notes, and references. +## Scope +- Harvest CERT-FR items via RSS and/or list pages; follow item pages for detail; window by publish/update date. +- Validate HTML or JSON payloads; extract structured fields; map to canonical aliases, references, severity text. +- Maintain watermarks and de-duplication by content hash; idempotent processing. +## Participants +- Source.Common (HTTP, HTML parsing helpers, validators). +- Storage.Mongo (document, dto, advisory, reference, source_state). +- Models (canonical). +- Core/WebService (jobs: source:certfr:fetch|parse|map). +- Merge engine (later) to enrich only. +## Interfaces & contracts +- Treat CERT-FR as enrichment; never override distro or PSIRT version ranges absent concrete evidence. +- References must include primary bulletin URL and vendor links; tag kind=bulletin/vendor/mitigation appropriately. +- Provenance records cite "cert-fr" with method=parser and source URL. +## In/Out of scope +In: advisory metadata extraction, references, severity text, watermarking. +Out: OVAL or package-level authority. +## Observability & security expectations +- Metrics: SourceDiagnostics emits shared `feedser.source.http.*` counters/histograms tagged `feedser.source=certfr`, covering fetch counts, parse failures, and map activity. +- Logs: feed URL(s), item ids/urls, extraction durations; no PII; allowlist hostnames. +## Tests +- Author and review coverage in `../StellaOps.Feedser.Source.CertFr.Tests`. +- Shared fixtures (e.g., `MongoIntegrationFixture`, `ConnectorTestHarness`) live in `../StellaOps.Feedser.Testing`. +- Keep fixtures deterministic; match new cases to real-world advisories or regression scenarios. diff --git a/src/StellaOps.Feedser.Source.CertFr/CertFrConnector.cs b/src/StellaOps.Feedser.Source.CertFr/CertFrConnector.cs new file mode 100644 index 00000000..264be9e2 --- /dev/null +++ b/src/StellaOps.Feedser.Source.CertFr/CertFrConnector.cs @@ -0,0 +1,337 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text.Json; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Options; +using MongoDB.Bson; +using StellaOps.Feedser.Source.CertFr.Configuration; +using StellaOps.Feedser.Source.CertFr.Internal; +using StellaOps.Feedser.Source.Common; +using StellaOps.Feedser.Source.Common.Fetch; +using StellaOps.Feedser.Storage.Mongo; +using StellaOps.Feedser.Storage.Mongo.Advisories; +using StellaOps.Feedser.Storage.Mongo.Documents; +using StellaOps.Feedser.Storage.Mongo.Dtos; +using StellaOps.Plugin; + +namespace StellaOps.Feedser.Source.CertFr; + +public sealed class CertFrConnector : IFeedConnector +{ + private static readonly JsonSerializerOptions SerializerOptions = new() + { + PropertyNamingPolicy = JsonNamingPolicy.CamelCase, + DefaultIgnoreCondition = System.Text.Json.Serialization.JsonIgnoreCondition.WhenWritingNull, + }; + + private readonly CertFrFeedClient _feedClient; + private readonly SourceFetchService _fetchService; + private readonly RawDocumentStorage _rawDocumentStorage; + private readonly IDocumentStore _documentStore; + private readonly IDtoStore _dtoStore; + private readonly IAdvisoryStore _advisoryStore; + private readonly ISourceStateRepository _stateRepository; + private readonly CertFrOptions _options; + private readonly TimeProvider _timeProvider; + private readonly ILogger _logger; + + public CertFrConnector( + CertFrFeedClient feedClient, + SourceFetchService fetchService, + RawDocumentStorage rawDocumentStorage, + IDocumentStore documentStore, + IDtoStore dtoStore, + IAdvisoryStore advisoryStore, + ISourceStateRepository stateRepository, + IOptions options, + TimeProvider? timeProvider, + ILogger logger) + { + _feedClient = feedClient ?? throw new ArgumentNullException(nameof(feedClient)); + _fetchService = fetchService ?? throw new ArgumentNullException(nameof(fetchService)); + _rawDocumentStorage = rawDocumentStorage ?? throw new ArgumentNullException(nameof(rawDocumentStorage)); + _documentStore = documentStore ?? throw new ArgumentNullException(nameof(documentStore)); + _dtoStore = dtoStore ?? throw new ArgumentNullException(nameof(dtoStore)); + _advisoryStore = advisoryStore ?? throw new ArgumentNullException(nameof(advisoryStore)); + _stateRepository = stateRepository ?? throw new ArgumentNullException(nameof(stateRepository)); + _options = (options ?? throw new ArgumentNullException(nameof(options))).Value ?? throw new ArgumentNullException(nameof(options)); + _options.Validate(); + _timeProvider = timeProvider ?? TimeProvider.System; + _logger = logger ?? throw new ArgumentNullException(nameof(logger)); + } + + public string SourceName => CertFrConnectorPlugin.SourceName; + + public async Task FetchAsync(IServiceProvider services, CancellationToken cancellationToken) + { + var cursor = await GetCursorAsync(cancellationToken).ConfigureAwait(false); + var now = _timeProvider.GetUtcNow(); + var windowEnd = now; + var lastPublished = cursor.LastPublished ?? now - _options.InitialBackfill; + var windowStart = lastPublished - _options.WindowOverlap; + var minStart = now - _options.InitialBackfill; + if (windowStart < minStart) + { + windowStart = minStart; + } + + IReadOnlyList items; + try + { + items = await _feedClient.LoadAsync(windowStart, windowEnd, cancellationToken).ConfigureAwait(false); + } + catch (Exception ex) + { + _logger.LogError(ex, "Cert-FR feed load failed {Start:o}-{End:o}", windowStart, windowEnd); + await _stateRepository.MarkFailureAsync(SourceName, now, TimeSpan.FromMinutes(10), ex.Message, cancellationToken).ConfigureAwait(false); + throw; + } + + if (items.Count == 0) + { + await UpdateCursorAsync(cursor.WithLastPublished(windowEnd), cancellationToken).ConfigureAwait(false); + return; + } + + var pendingDocuments = cursor.PendingDocuments.ToList(); + var pendingMappings = cursor.PendingMappings.ToList(); + var maxPublished = cursor.LastPublished ?? DateTimeOffset.MinValue; + + foreach (var item in items) + { + cancellationToken.ThrowIfCancellationRequested(); + + try + { + var existing = await _documentStore.FindBySourceAndUriAsync(SourceName, item.DetailUri.ToString(), cancellationToken).ConfigureAwait(false); + var request = new SourceFetchRequest(CertFrOptions.HttpClientName, SourceName, item.DetailUri) + { + Metadata = CertFrDocumentMetadata.CreateMetadata(item), + ETag = existing?.Etag, + LastModified = existing?.LastModified, + AcceptHeaders = new[] { "text/html", "application/xhtml+xml", "text/plain;q=0.5" }, + }; + + var result = await _fetchService.FetchAsync(request, cancellationToken).ConfigureAwait(false); + if (result.IsNotModified || !result.IsSuccess || result.Document is null) + { + if (item.Published > maxPublished) + { + maxPublished = item.Published; + } + + continue; + } + + if (existing is not null + && string.Equals(existing.Sha256, result.Document.Sha256, StringComparison.OrdinalIgnoreCase) + && string.Equals(existing.Status, DocumentStatuses.Mapped, StringComparison.Ordinal)) + { + await _documentStore.UpdateStatusAsync(result.Document.Id, existing.Status, cancellationToken).ConfigureAwait(false); + if (item.Published > maxPublished) + { + maxPublished = item.Published; + } + + continue; + } + + if (!pendingDocuments.Contains(result.Document.Id)) + { + pendingDocuments.Add(result.Document.Id); + } + + if (item.Published > maxPublished) + { + maxPublished = item.Published; + } + + if (_options.RequestDelay > TimeSpan.Zero) + { + await Task.Delay(_options.RequestDelay, cancellationToken).ConfigureAwait(false); + } + } + catch (Exception ex) + { + _logger.LogError(ex, "Cert-FR fetch failed for {Uri}", item.DetailUri); + await _stateRepository.MarkFailureAsync(SourceName, _timeProvider.GetUtcNow(), TimeSpan.FromMinutes(5), ex.Message, cancellationToken).ConfigureAwait(false); + throw; + } + } + + if (maxPublished == DateTimeOffset.MinValue) + { + maxPublished = cursor.LastPublished ?? windowEnd; + } + + var updatedCursor = cursor + .WithPendingDocuments(pendingDocuments) + .WithPendingMappings(pendingMappings) + .WithLastPublished(maxPublished); + + await UpdateCursorAsync(updatedCursor, cancellationToken).ConfigureAwait(false); + } + + public async Task ParseAsync(IServiceProvider services, CancellationToken cancellationToken) + { + var cursor = await GetCursorAsync(cancellationToken).ConfigureAwait(false); + if (cursor.PendingDocuments.Count == 0) + { + return; + } + + var pendingDocuments = cursor.PendingDocuments.ToList(); + var pendingMappings = cursor.PendingMappings.ToList(); + + foreach (var documentId in cursor.PendingDocuments) + { + cancellationToken.ThrowIfCancellationRequested(); + + var document = await _documentStore.FindAsync(documentId, cancellationToken).ConfigureAwait(false); + if (document is null) + { + pendingDocuments.Remove(documentId); + pendingMappings.Remove(documentId); + continue; + } + + if (!document.GridFsId.HasValue) + { + _logger.LogWarning("Cert-FR document {DocumentId} missing GridFS payload", document.Id); + await _documentStore.UpdateStatusAsync(document.Id, DocumentStatuses.Failed, cancellationToken).ConfigureAwait(false); + pendingDocuments.Remove(documentId); + pendingMappings.Remove(documentId); + continue; + } + + CertFrDocumentMetadata metadata; + try + { + metadata = CertFrDocumentMetadata.FromDocument(document); + } + catch (Exception ex) + { + _logger.LogError(ex, "Cert-FR metadata parse failed for document {DocumentId}", document.Id); + await _documentStore.UpdateStatusAsync(document.Id, DocumentStatuses.Failed, cancellationToken).ConfigureAwait(false); + pendingDocuments.Remove(documentId); + pendingMappings.Remove(documentId); + continue; + } + + CertFrDto dto; + try + { + var content = await _rawDocumentStorage.DownloadAsync(document.GridFsId.Value, cancellationToken).ConfigureAwait(false); + var html = System.Text.Encoding.UTF8.GetString(content); + dto = CertFrParser.Parse(html, metadata); + } + catch (Exception ex) + { + _logger.LogError(ex, "Cert-FR parse failed for advisory {AdvisoryId} ({Uri})", metadata.AdvisoryId, document.Uri); + await _documentStore.UpdateStatusAsync(document.Id, DocumentStatuses.Failed, cancellationToken).ConfigureAwait(false); + pendingDocuments.Remove(documentId); + pendingMappings.Remove(documentId); + continue; + } + + var json = JsonSerializer.Serialize(dto, SerializerOptions); + var payload = BsonDocument.Parse(json); + var validatedAt = _timeProvider.GetUtcNow(); + + var existingDto = await _dtoStore.FindByDocumentIdAsync(document.Id, cancellationToken).ConfigureAwait(false); + var dtoRecord = existingDto is null + ? new DtoRecord(Guid.NewGuid(), document.Id, SourceName, "certfr.detail.v1", payload, validatedAt) + : existingDto with + { + Payload = payload, + SchemaVersion = "certfr.detail.v1", + ValidatedAt = validatedAt, + }; + + await _dtoStore.UpsertAsync(dtoRecord, cancellationToken).ConfigureAwait(false); + await _documentStore.UpdateStatusAsync(document.Id, DocumentStatuses.PendingMap, cancellationToken).ConfigureAwait(false); + + pendingDocuments.Remove(documentId); + if (!pendingMappings.Contains(documentId)) + { + pendingMappings.Add(documentId); + } + } + + var updatedCursor = cursor + .WithPendingDocuments(pendingDocuments) + .WithPendingMappings(pendingMappings); + + await UpdateCursorAsync(updatedCursor, cancellationToken).ConfigureAwait(false); + } + + public async Task MapAsync(IServiceProvider services, CancellationToken cancellationToken) + { + var cursor = await GetCursorAsync(cancellationToken).ConfigureAwait(false); + if (cursor.PendingMappings.Count == 0) + { + return; + } + + var pendingMappings = cursor.PendingMappings.ToList(); + + foreach (var documentId in cursor.PendingMappings) + { + cancellationToken.ThrowIfCancellationRequested(); + + var dtoRecord = await _dtoStore.FindByDocumentIdAsync(documentId, cancellationToken).ConfigureAwait(false); + var document = await _documentStore.FindAsync(documentId, cancellationToken).ConfigureAwait(false); + + if (dtoRecord is null || document is null) + { + pendingMappings.Remove(documentId); + continue; + } + + CertFrDto? dto; + try + { + var json = dtoRecord.Payload.ToJson(); + dto = JsonSerializer.Deserialize(json, SerializerOptions); + } + catch (Exception ex) + { + _logger.LogError(ex, "Cert-FR DTO deserialization failed for document {DocumentId}", documentId); + await _documentStore.UpdateStatusAsync(document.Id, DocumentStatuses.Failed, cancellationToken).ConfigureAwait(false); + pendingMappings.Remove(documentId); + continue; + } + + if (dto is null) + { + _logger.LogWarning("Cert-FR DTO payload deserialized as null for document {DocumentId}", documentId); + await _documentStore.UpdateStatusAsync(document.Id, DocumentStatuses.Failed, cancellationToken).ConfigureAwait(false); + pendingMappings.Remove(documentId); + continue; + } + + var mappedAt = _timeProvider.GetUtcNow(); + var advisory = CertFrMapper.Map(dto, SourceName, mappedAt); + await _advisoryStore.UpsertAsync(advisory, cancellationToken).ConfigureAwait(false); + await _documentStore.UpdateStatusAsync(document.Id, DocumentStatuses.Mapped, cancellationToken).ConfigureAwait(false); + + pendingMappings.Remove(documentId); + } + + var updatedCursor = cursor.WithPendingMappings(pendingMappings); + await UpdateCursorAsync(updatedCursor, cancellationToken).ConfigureAwait(false); + } + + private async Task GetCursorAsync(CancellationToken cancellationToken) + { + var record = await _stateRepository.TryGetAsync(SourceName, cancellationToken).ConfigureAwait(false); + return CertFrCursor.FromBson(record?.Cursor); + } + + private async Task UpdateCursorAsync(CertFrCursor cursor, CancellationToken cancellationToken) + { + var completedAt = _timeProvider.GetUtcNow(); + await _stateRepository.UpdateCursorAsync(SourceName, cursor.ToBsonDocument(), completedAt, cancellationToken).ConfigureAwait(false); + } +} diff --git a/src/StellaOps.Feedser.Source.CertFr/CertFrConnectorPlugin.cs b/src/StellaOps.Feedser.Source.CertFr/CertFrConnectorPlugin.cs new file mode 100644 index 00000000..fb2357aa --- /dev/null +++ b/src/StellaOps.Feedser.Source.CertFr/CertFrConnectorPlugin.cs @@ -0,0 +1,21 @@ +using System; +using Microsoft.Extensions.DependencyInjection; +using StellaOps.Plugin; + +namespace StellaOps.Feedser.Source.CertFr; + +public sealed class CertFrConnectorPlugin : IConnectorPlugin +{ + public const string SourceName = "cert-fr"; + + public string Name => SourceName; + + public bool IsAvailable(IServiceProvider services) + => services.GetService() is not null; + + public IFeedConnector Create(IServiceProvider services) + { + ArgumentNullException.ThrowIfNull(services); + return services.GetRequiredService(); + } +} diff --git a/src/StellaOps.Feedser.Source.CertFr/CertFrDependencyInjectionRoutine.cs b/src/StellaOps.Feedser.Source.CertFr/CertFrDependencyInjectionRoutine.cs new file mode 100644 index 00000000..9effd14d --- /dev/null +++ b/src/StellaOps.Feedser.Source.CertFr/CertFrDependencyInjectionRoutine.cs @@ -0,0 +1,54 @@ +using System; +using Microsoft.Extensions.Configuration; +using Microsoft.Extensions.DependencyInjection; +using StellaOps.DependencyInjection; +using StellaOps.Feedser.Core.Jobs; +using StellaOps.Feedser.Source.CertFr.Configuration; + +namespace StellaOps.Feedser.Source.CertFr; + +public sealed class CertFrDependencyInjectionRoutine : IDependencyInjectionRoutine +{ + private const string ConfigurationSection = "feedser:sources:cert-fr"; + + public IServiceCollection Register(IServiceCollection services, IConfiguration configuration) + { + ArgumentNullException.ThrowIfNull(services); + ArgumentNullException.ThrowIfNull(configuration); + + services.AddCertFrConnector(options => + { + configuration.GetSection(ConfigurationSection).Bind(options); + options.Validate(); + }); + + services.AddTransient(); + services.AddTransient(); + services.AddTransient(); + + services.PostConfigure(options => + { + EnsureJob(options, CertFrJobKinds.Fetch, typeof(CertFrFetchJob)); + EnsureJob(options, CertFrJobKinds.Parse, typeof(CertFrParseJob)); + EnsureJob(options, CertFrJobKinds.Map, typeof(CertFrMapJob)); + }); + + return services; + } + + private static void EnsureJob(JobSchedulerOptions options, string kind, Type jobType) + { + if (options.Definitions.ContainsKey(kind)) + { + return; + } + + options.Definitions[kind] = new JobDefinition( + kind, + jobType, + options.DefaultTimeout, + options.DefaultLeaseDuration, + CronExpression: null, + Enabled: true); + } +} diff --git a/src/StellaOps.Feedser.Source.CertFr/CertFrServiceCollectionExtensions.cs b/src/StellaOps.Feedser.Source.CertFr/CertFrServiceCollectionExtensions.cs new file mode 100644 index 00000000..80446c97 --- /dev/null +++ b/src/StellaOps.Feedser.Source.CertFr/CertFrServiceCollectionExtensions.cs @@ -0,0 +1,36 @@ +using System; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.DependencyInjection.Extensions; +using Microsoft.Extensions.Options; +using StellaOps.Feedser.Source.CertFr.Configuration; +using StellaOps.Feedser.Source.CertFr.Internal; +using StellaOps.Feedser.Source.Common.Http; + +namespace StellaOps.Feedser.Source.CertFr; + +public static class CertFrServiceCollectionExtensions +{ + public static IServiceCollection AddCertFrConnector(this IServiceCollection services, Action configure) + { + ArgumentNullException.ThrowIfNull(services); + ArgumentNullException.ThrowIfNull(configure); + + services.AddOptions() + .Configure(configure) + .PostConfigure(static options => options.Validate()); + + services.AddSourceHttpClient(CertFrOptions.HttpClientName, static (sp, clientOptions) => + { + var options = sp.GetRequiredService>().Value; + clientOptions.BaseAddress = options.FeedUri; + clientOptions.UserAgent = "StellaOps.Feedser.CertFr/1.0"; + clientOptions.Timeout = TimeSpan.FromSeconds(20); + clientOptions.AllowedHosts.Clear(); + clientOptions.AllowedHosts.Add(options.FeedUri.Host); + }); + + services.TryAddSingleton(); + services.AddTransient(); + return services; + } +} diff --git a/src/StellaOps.Feedser.Source.CertFr/Configuration/CertFrOptions.cs b/src/StellaOps.Feedser.Source.CertFr/Configuration/CertFrOptions.cs new file mode 100644 index 00000000..0593204f --- /dev/null +++ b/src/StellaOps.Feedser.Source.CertFr/Configuration/CertFrOptions.cs @@ -0,0 +1,46 @@ +using System; + +namespace StellaOps.Feedser.Source.CertFr.Configuration; + +public sealed class CertFrOptions +{ + public const string HttpClientName = "cert-fr"; + + public Uri FeedUri { get; set; } = new("https://www.cert.ssi.gouv.fr/feed/alertes/"); + + public TimeSpan InitialBackfill { get; set; } = TimeSpan.FromDays(30); + + public TimeSpan WindowOverlap { get; set; } = TimeSpan.FromDays(2); + + public int MaxItemsPerFetch { get; set; } = 100; + + public TimeSpan RequestDelay { get; set; } = TimeSpan.Zero; + + public void Validate() + { + if (FeedUri is null || !FeedUri.IsAbsoluteUri) + { + throw new InvalidOperationException("Cert-FR FeedUri must be an absolute URI."); + } + + if (InitialBackfill <= TimeSpan.Zero) + { + throw new InvalidOperationException("InitialBackfill must be a positive duration."); + } + + if (WindowOverlap < TimeSpan.Zero) + { + throw new InvalidOperationException("WindowOverlap cannot be negative."); + } + + if (MaxItemsPerFetch <= 0) + { + throw new InvalidOperationException("MaxItemsPerFetch must be positive."); + } + + if (RequestDelay < TimeSpan.Zero) + { + throw new InvalidOperationException("RequestDelay cannot be negative."); + } + } +} diff --git a/src/StellaOps.Feedser.Source.CertFr/Internal/CertFrCursor.cs b/src/StellaOps.Feedser.Source.CertFr/Internal/CertFrCursor.cs new file mode 100644 index 00000000..434ce028 --- /dev/null +++ b/src/StellaOps.Feedser.Source.CertFr/Internal/CertFrCursor.cs @@ -0,0 +1,88 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using MongoDB.Bson; + +namespace StellaOps.Feedser.Source.CertFr.Internal; + +internal sealed record CertFrCursor( + DateTimeOffset? LastPublished, + IReadOnlyCollection PendingDocuments, + IReadOnlyCollection PendingMappings) +{ + public static CertFrCursor Empty { get; } = new(null, Array.Empty(), Array.Empty()); + + public BsonDocument ToBsonDocument() + { + var document = new BsonDocument + { + ["pendingDocuments"] = new BsonArray(PendingDocuments.Select(id => id.ToString())), + ["pendingMappings"] = new BsonArray(PendingMappings.Select(id => id.ToString())), + }; + + if (LastPublished.HasValue) + { + document["lastPublished"] = LastPublished.Value.UtcDateTime; + } + + return document; + } + + public static CertFrCursor FromBson(BsonDocument? document) + { + if (document is null || document.ElementCount == 0) + { + return Empty; + } + + var lastPublished = document.TryGetValue("lastPublished", out var value) + ? ParseDate(value) + : null; + + return new CertFrCursor( + lastPublished, + ReadGuidArray(document, "pendingDocuments"), + ReadGuidArray(document, "pendingMappings")); + } + + public CertFrCursor WithLastPublished(DateTimeOffset? timestamp) + => this with { LastPublished = timestamp }; + + public CertFrCursor WithPendingDocuments(IEnumerable ids) + => this with { PendingDocuments = ids?.Distinct().ToArray() ?? Array.Empty() }; + + public CertFrCursor WithPendingMappings(IEnumerable ids) + => this with { PendingMappings = ids?.Distinct().ToArray() ?? Array.Empty() }; + + private static DateTimeOffset? ParseDate(BsonValue value) + => value.BsonType switch + { + BsonType.DateTime => DateTime.SpecifyKind(value.ToUniversalTime(), DateTimeKind.Utc), + BsonType.String when DateTimeOffset.TryParse(value.AsString, out var parsed) => parsed.ToUniversalTime(), + _ => null, + }; + + private static IReadOnlyCollection ReadGuidArray(BsonDocument document, string field) + { + if (!document.TryGetValue(field, out var raw) || raw is not BsonArray array) + { + return Array.Empty(); + } + + var result = new List(array.Count); + foreach (var element in array) + { + if (element is null) + { + continue; + } + + if (Guid.TryParse(element.ToString(), out var guid)) + { + result.Add(guid); + } + } + + return result; + } +} diff --git a/src/StellaOps.Feedser.Source.CertFr/Internal/CertFrDocumentMetadata.cs b/src/StellaOps.Feedser.Source.CertFr/Internal/CertFrDocumentMetadata.cs new file mode 100644 index 00000000..d8bd1d35 --- /dev/null +++ b/src/StellaOps.Feedser.Source.CertFr/Internal/CertFrDocumentMetadata.cs @@ -0,0 +1,77 @@ +using System; +using System.Collections.Generic; +using StellaOps.Feedser.Storage.Mongo.Documents; + +namespace StellaOps.Feedser.Source.CertFr.Internal; + +internal sealed record CertFrDocumentMetadata( + string AdvisoryId, + string Title, + DateTimeOffset Published, + Uri DetailUri, + string? Summary) +{ + private const string AdvisoryIdKey = "certfr.advisoryId"; + private const string TitleKey = "certfr.title"; + private const string PublishedKey = "certfr.published"; + private const string SummaryKey = "certfr.summary"; + + public static CertFrDocumentMetadata FromDocument(DocumentRecord document) + { + ArgumentNullException.ThrowIfNull(document); + + if (document.Metadata is null) + { + throw new InvalidOperationException("Cert-FR document metadata is missing."); + } + + var metadata = document.Metadata; + if (!metadata.TryGetValue(AdvisoryIdKey, out var advisoryId) || string.IsNullOrWhiteSpace(advisoryId)) + { + throw new InvalidOperationException("Cert-FR advisory id metadata missing."); + } + + if (!metadata.TryGetValue(TitleKey, out var title) || string.IsNullOrWhiteSpace(title)) + { + throw new InvalidOperationException("Cert-FR title metadata missing."); + } + + if (!metadata.TryGetValue(PublishedKey, out var publishedRaw) || !DateTimeOffset.TryParse(publishedRaw, out var published)) + { + throw new InvalidOperationException("Cert-FR published metadata invalid."); + } + + if (!Uri.TryCreate(document.Uri, UriKind.Absolute, out var detailUri)) + { + throw new InvalidOperationException("Cert-FR document URI invalid."); + } + + metadata.TryGetValue(SummaryKey, out var summary); + + return new CertFrDocumentMetadata( + advisoryId.Trim(), + title.Trim(), + published.ToUniversalTime(), + detailUri, + string.IsNullOrWhiteSpace(summary) ? null : summary.Trim()); + } + + public static IReadOnlyDictionary CreateMetadata(CertFrFeedItem item) + { + ArgumentNullException.ThrowIfNull(item); + + var metadata = new Dictionary(StringComparer.Ordinal) + { + [AdvisoryIdKey] = item.AdvisoryId, + [TitleKey] = item.Title ?? item.AdvisoryId, + [PublishedKey] = item.Published.ToString("O"), + }; + + if (!string.IsNullOrWhiteSpace(item.Summary)) + { + metadata[SummaryKey] = item.Summary!; + } + + return metadata; + } +} diff --git a/src/StellaOps.Feedser.Source.CertFr/Internal/CertFrDto.cs b/src/StellaOps.Feedser.Source.CertFr/Internal/CertFrDto.cs new file mode 100644 index 00000000..9b25fea1 --- /dev/null +++ b/src/StellaOps.Feedser.Source.CertFr/Internal/CertFrDto.cs @@ -0,0 +1,14 @@ +using System; +using System.Collections.Generic; +using System.Text.Json.Serialization; + +namespace StellaOps.Feedser.Source.CertFr.Internal; + +internal sealed record CertFrDto( + [property: JsonPropertyName("advisoryId")] string AdvisoryId, + [property: JsonPropertyName("title")] string Title, + [property: JsonPropertyName("detailUrl")] string DetailUrl, + [property: JsonPropertyName("published")] DateTimeOffset Published, + [property: JsonPropertyName("summary")] string? Summary, + [property: JsonPropertyName("content")] string Content, + [property: JsonPropertyName("references")] IReadOnlyList References); diff --git a/src/StellaOps.Feedser.Source.CertFr/Internal/CertFrFeedClient.cs b/src/StellaOps.Feedser.Source.CertFr/Internal/CertFrFeedClient.cs new file mode 100644 index 00000000..b160f11b --- /dev/null +++ b/src/StellaOps.Feedser.Source.CertFr/Internal/CertFrFeedClient.cs @@ -0,0 +1,109 @@ +using System; +using System.Collections.Generic; +using System.Globalization; +using System.Linq; +using System.Net.Http; +using System.Threading; +using System.Threading.Tasks; +using System.Xml.Linq; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Options; +using StellaOps.Feedser.Source.CertFr.Configuration; + +namespace StellaOps.Feedser.Source.CertFr.Internal; + +public sealed class CertFrFeedClient +{ + private readonly IHttpClientFactory _httpClientFactory; + private readonly CertFrOptions _options; + private readonly ILogger _logger; + + public CertFrFeedClient(IHttpClientFactory httpClientFactory, IOptions options, ILogger logger) + { + _httpClientFactory = httpClientFactory ?? throw new ArgumentNullException(nameof(httpClientFactory)); + _options = (options ?? throw new ArgumentNullException(nameof(options))).Value ?? throw new ArgumentNullException(nameof(options)); + _options.Validate(); + _logger = logger ?? throw new ArgumentNullException(nameof(logger)); + } + + public async Task> LoadAsync(DateTimeOffset windowStart, DateTimeOffset windowEnd, CancellationToken cancellationToken) + { + var client = _httpClientFactory.CreateClient(CertFrOptions.HttpClientName); + + using var response = await client.GetAsync(_options.FeedUri, cancellationToken).ConfigureAwait(false); + response.EnsureSuccessStatusCode(); + + await using var stream = await response.Content.ReadAsStreamAsync(cancellationToken).ConfigureAwait(false); + var document = XDocument.Load(stream); + + var items = new List(); + var now = DateTimeOffset.UtcNow; + + foreach (var itemElement in document.Descendants("item")) + { + var link = itemElement.Element("link")?.Value; + if (string.IsNullOrWhiteSpace(link) || !Uri.TryCreate(link.Trim(), UriKind.Absolute, out var detailUri)) + { + continue; + } + + var title = itemElement.Element("title")?.Value?.Trim(); + var summary = itemElement.Element("description")?.Value?.Trim(); + + var published = ParsePublished(itemElement.Element("pubDate")?.Value) ?? now; + if (published < windowStart) + { + continue; + } + + if (published > windowEnd) + { + published = windowEnd; + } + + var advisoryId = ResolveAdvisoryId(itemElement, detailUri); + items.Add(new CertFrFeedItem(advisoryId, detailUri, published.ToUniversalTime(), title, summary)); + } + + return items + .OrderBy(item => item.Published) + .Take(_options.MaxItemsPerFetch) + .ToArray(); + } + + private static DateTimeOffset? ParsePublished(string? value) + { + if (string.IsNullOrWhiteSpace(value)) + { + return null; + } + + if (DateTimeOffset.TryParse(value, CultureInfo.InvariantCulture, DateTimeStyles.AssumeUniversal | DateTimeStyles.AdjustToUniversal, out var parsed)) + { + return parsed; + } + + return null; + } + + private static string ResolveAdvisoryId(XElement itemElement, Uri detailUri) + { + var guid = itemElement.Element("guid")?.Value; + if (!string.IsNullOrWhiteSpace(guid)) + { + return guid.Trim(); + } + + var segments = detailUri.Segments; + if (segments.Length > 0) + { + var slug = segments[^1].Trim('/'); + if (!string.IsNullOrWhiteSpace(slug)) + { + return slug; + } + } + + return detailUri.AbsoluteUri; + } +} diff --git a/src/StellaOps.Feedser.Source.CertFr/Internal/CertFrFeedItem.cs b/src/StellaOps.Feedser.Source.CertFr/Internal/CertFrFeedItem.cs new file mode 100644 index 00000000..3e222528 --- /dev/null +++ b/src/StellaOps.Feedser.Source.CertFr/Internal/CertFrFeedItem.cs @@ -0,0 +1,10 @@ +using System; + +namespace StellaOps.Feedser.Source.CertFr.Internal; + +public sealed record CertFrFeedItem( + string AdvisoryId, + Uri DetailUri, + DateTimeOffset Published, + string? Title, + string? Summary); diff --git a/src/StellaOps.Feedser.Source.CertFr/Internal/CertFrMapper.cs b/src/StellaOps.Feedser.Source.CertFr/Internal/CertFrMapper.cs new file mode 100644 index 00000000..eb72ccd8 --- /dev/null +++ b/src/StellaOps.Feedser.Source.CertFr/Internal/CertFrMapper.cs @@ -0,0 +1,116 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using StellaOps.Feedser.Models; + +namespace StellaOps.Feedser.Source.CertFr.Internal; + +internal static class CertFrMapper +{ + public static Advisory Map(CertFrDto dto, string sourceName, DateTimeOffset recordedAt) + { + ArgumentNullException.ThrowIfNull(dto); + ArgumentException.ThrowIfNullOrEmpty(sourceName); + + var advisoryKey = $"cert-fr/{dto.AdvisoryId}"; + var provenance = new AdvisoryProvenance(sourceName, "document", dto.DetailUrl, recordedAt.ToUniversalTime()); + + var aliases = new List + { + $"CERT-FR:{dto.AdvisoryId}", + }; + + var references = BuildReferences(dto, provenance).ToArray(); + + var affectedPackages = BuildAffectedPackages(dto, provenance).ToArray(); + + return new Advisory( + advisoryKey, + dto.Title, + dto.Summary ?? dto.Title, + language: "fr", + published: dto.Published.ToUniversalTime(), + modified: null, + severity: null, + exploitKnown: false, + aliases: aliases, + references: references, + affectedPackages: affectedPackages, + cvssMetrics: Array.Empty(), + provenance: new[] { provenance }); + } + + private static IEnumerable BuildReferences(CertFrDto dto, AdvisoryProvenance provenance) + { + var comparer = StringComparer.OrdinalIgnoreCase; + var entries = new List<(AdvisoryReference Reference, int Priority)> + { + (new AdvisoryReference(dto.DetailUrl, "advisory", "cert-fr", dto.Summary, provenance), 0), + }; + + foreach (var url in dto.References) + { + entries.Add((new AdvisoryReference(url, "reference", null, null, provenance), 1)); + } + + return entries + .GroupBy(tuple => tuple.Reference.Url, comparer) + .Select(group => group + .OrderBy(t => t.Priority) + .ThenBy(t => t.Reference.Kind ?? string.Empty, comparer) + .ThenBy(t => t.Reference.Url, comparer) + .First()) + .OrderBy(t => t.Priority) + .ThenBy(t => t.Reference.Kind ?? string.Empty, comparer) + .ThenBy(t => t.Reference.Url, comparer) + .Select(t => t.Reference); + } + + private static IEnumerable BuildAffectedPackages(CertFrDto dto, AdvisoryProvenance provenance) + { + var extensions = new Dictionary(StringComparer.OrdinalIgnoreCase); + if (!string.IsNullOrWhiteSpace(dto.Summary)) + { + extensions["certfr.summary"] = dto.Summary.Trim(); + } + + if (!string.IsNullOrWhiteSpace(dto.Content)) + { + var trimmed = dto.Content.Length > 1024 ? dto.Content[..1024].Trim() : dto.Content.Trim(); + if (trimmed.Length > 0) + { + extensions["certfr.content"] = trimmed; + } + } + + if (dto.References.Count > 0) + { + extensions["certfr.reference.count"] = dto.References.Count.ToString(); + } + + if (extensions.Count == 0) + { + return Array.Empty(); + } + + var range = new AffectedVersionRange( + rangeKind: "vendor", + introducedVersion: null, + fixedVersion: null, + lastAffectedVersion: null, + rangeExpression: null, + provenance: provenance, + primitives: new RangePrimitives(null, null, null, extensions)); + + return new[] + { + new AffectedPackage( + AffectedPackageTypes.Vendor, + identifier: dto.AdvisoryId, + platform: null, + versionRanges: new[] { range }, + statuses: Array.Empty(), + provenance: new[] { provenance }) + }; + } +} diff --git a/src/StellaOps.Feedser.Source.CertFr/Internal/CertFrParser.cs b/src/StellaOps.Feedser.Source.CertFr/Internal/CertFrParser.cs new file mode 100644 index 00000000..48a87520 --- /dev/null +++ b/src/StellaOps.Feedser.Source.CertFr/Internal/CertFrParser.cs @@ -0,0 +1,80 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text.RegularExpressions; + +namespace StellaOps.Feedser.Source.CertFr.Internal; + +internal static class CertFrParser +{ + private static readonly Regex AnchorRegex = new("]+href=\"(?https?://[^\"]+)\"", RegexOptions.IgnoreCase | RegexOptions.Compiled); + private static readonly Regex ScriptRegex = new("", RegexOptions.IgnoreCase | RegexOptions.Compiled); + private static readonly Regex StyleRegex = new("", RegexOptions.IgnoreCase | RegexOptions.Compiled); + private static readonly Regex TagRegex = new("<[^>]+>", RegexOptions.Compiled); + private static readonly Regex WhitespaceRegex = new("\\s+", RegexOptions.Compiled); + + public static CertFrDto Parse(string html, CertFrDocumentMetadata metadata) + { + ArgumentException.ThrowIfNullOrEmpty(html); + ArgumentNullException.ThrowIfNull(metadata); + + var sanitized = SanitizeHtml(html); + var summary = BuildSummary(metadata.Summary, sanitized); + var references = ExtractReferences(html); + + return new CertFrDto( + metadata.AdvisoryId, + metadata.Title, + metadata.DetailUri.ToString(), + metadata.Published, + summary, + sanitized, + references); + } + + private static string SanitizeHtml(string html) + { + var withoutScripts = ScriptRegex.Replace(html, string.Empty); + var withoutStyles = StyleRegex.Replace(withoutScripts, string.Empty); + var withoutTags = TagRegex.Replace(withoutStyles, " "); + var decoded = System.Net.WebUtility.HtmlDecode(withoutTags) ?? string.Empty; + return WhitespaceRegex.Replace(decoded, " ").Trim(); + } + + private static string? BuildSummary(string? metadataSummary, string content) + { + if (!string.IsNullOrWhiteSpace(metadataSummary)) + { + return metadataSummary.Trim(); + } + + if (string.IsNullOrWhiteSpace(content)) + { + return null; + } + + var sentences = content.Split(new[] { '.','!','?' }, StringSplitOptions.RemoveEmptyEntries | StringSplitOptions.TrimEntries); + if (sentences.Length > 0) + { + return sentences[0].Trim(); + } + + return content.Length > 280 ? content[..280].Trim() : content; + } + + private static IReadOnlyList ExtractReferences(string html) + { + var references = new HashSet(StringComparer.OrdinalIgnoreCase); + foreach (Match match in AnchorRegex.Matches(html)) + { + if (match.Success) + { + references.Add(match.Groups["url"].Value.Trim()); + } + } + + return references.Count == 0 + ? Array.Empty() + : references.OrderBy(url => url, StringComparer.OrdinalIgnoreCase).ToArray(); + } +} diff --git a/src/StellaOps.Feedser.Source.CertFr/Jobs.cs b/src/StellaOps.Feedser.Source.CertFr/Jobs.cs new file mode 100644 index 00000000..6994fe84 --- /dev/null +++ b/src/StellaOps.Feedser.Source.CertFr/Jobs.cs @@ -0,0 +1,46 @@ +using System; +using System.Threading; +using System.Threading.Tasks; +using StellaOps.Feedser.Core.Jobs; + +namespace StellaOps.Feedser.Source.CertFr; + +internal static class CertFrJobKinds +{ + public const string Fetch = "source:cert-fr:fetch"; + public const string Parse = "source:cert-fr:parse"; + public const string Map = "source:cert-fr:map"; +} + +internal sealed class CertFrFetchJob : IJob +{ + private readonly CertFrConnector _connector; + + public CertFrFetchJob(CertFrConnector connector) + => _connector = connector ?? throw new ArgumentNullException(nameof(connector)); + + public Task ExecuteAsync(JobExecutionContext context, CancellationToken cancellationToken) + => _connector.FetchAsync(context.Services, cancellationToken); +} + +internal sealed class CertFrParseJob : IJob +{ + private readonly CertFrConnector _connector; + + public CertFrParseJob(CertFrConnector connector) + => _connector = connector ?? throw new ArgumentNullException(nameof(connector)); + + public Task ExecuteAsync(JobExecutionContext context, CancellationToken cancellationToken) + => _connector.ParseAsync(context.Services, cancellationToken); +} + +internal sealed class CertFrMapJob : IJob +{ + private readonly CertFrConnector _connector; + + public CertFrMapJob(CertFrConnector connector) + => _connector = connector ?? throw new ArgumentNullException(nameof(connector)); + + public Task ExecuteAsync(JobExecutionContext context, CancellationToken cancellationToken) + => _connector.MapAsync(context.Services, cancellationToken); +} diff --git a/src/StellaOps.Feedser.Source.CertFr/StellaOps.Feedser.Source.CertFr.csproj b/src/StellaOps.Feedser.Source.CertFr/StellaOps.Feedser.Source.CertFr.csproj new file mode 100644 index 00000000..a01e6075 --- /dev/null +++ b/src/StellaOps.Feedser.Source.CertFr/StellaOps.Feedser.Source.CertFr.csproj @@ -0,0 +1,13 @@ + + + net10.0 + enable + enable + + + + + + + + diff --git a/src/StellaOps.Feedser.Source.CertFr/TASKS.md b/src/StellaOps.Feedser.Source.CertFr/TASKS.md new file mode 100644 index 00000000..c15ae640 --- /dev/null +++ b/src/StellaOps.Feedser.Source.CertFr/TASKS.md @@ -0,0 +1,11 @@ +# TASKS +| Task | Owner(s) | Depends on | Notes | +|---|---|---|---| +|RSS/list fetcher with sliding window|BE-Conn-CertFr|Source.Common|**DONE** – RSS/list ingestion implemented with sliding date cursor.| +|Detail page fetch and sanitizer|BE-Conn-CertFr|Source.Common|**DONE** – HTML sanitizer trims boilerplate prior to DTO mapping.| +|Extractor and schema validation of DTO|BE-Conn-CertFr, QA|Source.Common|**DONE** – DTO parsing validates structure before persistence.| +|Canonical mapping (aliases, refs, severity text)|BE-Conn-CertFr|Models|**DONE** – mapper emits enrichment references with severity text.| +|Watermark plus dedupe by sha256|BE-Conn-CertFr|Storage.Mongo|**DONE** – SHA comparisons skip unchanged docs; covered by duplicate/not-modified connector tests.| +|Golden fixtures and determinism tests|QA|Source.CertFr|**DONE** – snapshot fixtures added in `CertFrConnectorTests` to enforce deterministic output.| +|Mark failure/backoff on fetch errors|BE-Conn-CertFr|Storage.Mongo|**DONE** – fetch path now marks failures/backoff and tests assert state repository updates.| +|Conditional fetch caching|BE-Conn-CertFr|Source.Common|**DONE** – ETag/Last-Modified support wired via `SourceFetchService` and verified in not-modified test.| diff --git a/src/StellaOps.Feedser.Source.CertIn.Tests/CertIn/CertInConnectorTests.cs b/src/StellaOps.Feedser.Source.CertIn.Tests/CertIn/CertInConnectorTests.cs new file mode 100644 index 00000000..56dcabc1 --- /dev/null +++ b/src/StellaOps.Feedser.Source.CertIn.Tests/CertIn/CertInConnectorTests.cs @@ -0,0 +1,350 @@ +using System.Collections.Generic; +using System.Globalization; +using System.IO; +using System.Net; +using System.Net.Http; +using System.Net.Http.Headers; +using System.Text; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Http; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Logging.Abstractions; +using Microsoft.Extensions.Options; +using Microsoft.Extensions.Time.Testing; +using MongoDB.Bson; +using StellaOps.Feedser.Models; +using StellaOps.Feedser.Source.CertIn; +using StellaOps.Feedser.Source.CertIn.Configuration; +using StellaOps.Feedser.Source.CertIn.Internal; +using StellaOps.Feedser.Source.Common; +using StellaOps.Feedser.Source.Common.Fetch; +using StellaOps.Feedser.Source.Common.Http; +using StellaOps.Feedser.Source.Common.Testing; +using StellaOps.Feedser.Storage.Mongo; +using StellaOps.Feedser.Storage.Mongo.Advisories; +using StellaOps.Feedser.Storage.Mongo.Documents; +using StellaOps.Feedser.Storage.Mongo.Dtos; +using StellaOps.Feedser.Testing; + +namespace StellaOps.Feedser.Source.CertIn.Tests; + +[Collection("mongo-fixture")] +public sealed class CertInConnectorTests : IAsyncLifetime +{ + private readonly MongoIntegrationFixture _fixture; + private readonly FakeTimeProvider _timeProvider; + private readonly CannedHttpMessageHandler _handler; + private ServiceProvider? _serviceProvider; + + public CertInConnectorTests(MongoIntegrationFixture fixture) + { + _fixture = fixture; + _timeProvider = new FakeTimeProvider(new DateTimeOffset(2024, 4, 20, 0, 0, 0, TimeSpan.Zero)); + _handler = new CannedHttpMessageHandler(); + } + + [Fact] + public async Task FetchParseMap_GeneratesExpectedSnapshot() + { + var options = new CertInOptions + { + AlertsEndpoint = new Uri("https://cert-in.example/api/alerts", UriKind.Absolute), + WindowSize = TimeSpan.FromDays(60), + WindowOverlap = TimeSpan.FromDays(7), + MaxPagesPerFetch = 1, + RequestDelay = TimeSpan.Zero, + }; + + await EnsureServiceProviderAsync(options); + var provider = _serviceProvider!; + + _handler.Clear(); + + _handler.AddTextResponse(options.AlertsEndpoint, ReadFixture("alerts-page1.json"), "application/json"); + var detailUri = new Uri("https://cert-in.example/advisory/CIAD-2024-0005"); + _handler.AddTextResponse(detailUri, ReadFixture("detail-CIAD-2024-0005.html"), "text/html"); + + var connector = new CertInConnectorPlugin().Create(provider); + + await connector.FetchAsync(provider, CancellationToken.None); + + _timeProvider.Advance(TimeSpan.FromMinutes(1)); + await connector.ParseAsync(provider, CancellationToken.None); + + await connector.MapAsync(provider, CancellationToken.None); + + var advisoryStore = provider.GetRequiredService(); + var advisories = await advisoryStore.GetRecentAsync(5, CancellationToken.None); + Assert.Single(advisories); + var canonical = SnapshotSerializer.ToSnapshot(advisories.Single()); + var expected = ReadFixture("expected-advisory.json"); + var normalizedExpected = NormalizeLineEndings(expected); + var normalizedActual = NormalizeLineEndings(canonical); + if (!string.Equals(normalizedExpected, normalizedActual, StringComparison.Ordinal)) + { + var actualPath = ResolveFixturePath("expected-advisory.actual.json"); + Directory.CreateDirectory(Path.GetDirectoryName(actualPath)!); + File.WriteAllText(actualPath, canonical); + } + + Assert.Equal(normalizedExpected, normalizedActual); + + var documentStore = provider.GetRequiredService(); + var document = await documentStore.FindBySourceAndUriAsync(CertInConnectorPlugin.SourceName, detailUri.ToString(), CancellationToken.None); + Assert.NotNull(document); + Assert.Equal(DocumentStatuses.Mapped, document!.Status); + + var stateRepository = provider.GetRequiredService(); + var state = await stateRepository.TryGetAsync(CertInConnectorPlugin.SourceName, CancellationToken.None); + Assert.NotNull(state); + Assert.True(state!.Cursor.TryGetValue("pendingDocuments", out var pending)); + Assert.Empty(pending.AsBsonArray); + } + + [Fact] + public async Task FetchFailure_RecordsBackoffAndReason() + { + var options = new CertInOptions + { + AlertsEndpoint = new Uri("https://cert-in.example/api/alerts", UriKind.Absolute), + WindowSize = TimeSpan.FromDays(60), + WindowOverlap = TimeSpan.FromDays(7), + MaxPagesPerFetch = 1, + RequestDelay = TimeSpan.Zero, + }; + + await EnsureServiceProviderAsync(options); + _handler.Clear(); + _handler.AddResponse(options.AlertsEndpoint, () => new HttpResponseMessage(HttpStatusCode.InternalServerError) + { + Content = new StringContent("{}", Encoding.UTF8, "application/json"), + }); + + var provider = _serviceProvider!; + var connector = new CertInConnectorPlugin().Create(provider); + + await Assert.ThrowsAsync(() => connector.FetchAsync(provider, CancellationToken.None)); + + var stateRepository = provider.GetRequiredService(); + var state = await stateRepository.TryGetAsync(CertInConnectorPlugin.SourceName, CancellationToken.None); + Assert.NotNull(state); + Assert.Equal(1, state!.FailCount); + Assert.NotNull(state.LastFailureReason); + Assert.Contains("500", state.LastFailureReason, StringComparison.Ordinal); + Assert.True(state.BackoffUntil.HasValue); + Assert.True(state.BackoffUntil!.Value > _timeProvider.GetUtcNow()); + } + + [Fact] + public async Task Fetch_NotModifiedMaintainsDocumentState() + { + var options = new CertInOptions + { + AlertsEndpoint = new Uri("https://cert-in.example/api/alerts", UriKind.Absolute), + WindowSize = TimeSpan.FromDays(30), + WindowOverlap = TimeSpan.FromDays(7), + MaxPagesPerFetch = 1, + RequestDelay = TimeSpan.Zero, + }; + + await EnsureServiceProviderAsync(options); + var provider = _serviceProvider!; + _handler.Clear(); + + var listingPayload = ReadFixture("alerts-page1.json"); + var detailUri = new Uri("https://cert-in.example/advisory/CIAD-2024-0005"); + var detailHtml = ReadFixture("detail-CIAD-2024-0005.html"); + var etag = new EntityTagHeaderValue("\"certin-2024-0005\""); + var lastModified = new DateTimeOffset(2024, 4, 15, 10, 0, 0, TimeSpan.Zero); + + _handler.AddTextResponse(options.AlertsEndpoint, listingPayload, "application/json"); + _handler.AddResponse(detailUri, () => + { + var response = new HttpResponseMessage(HttpStatusCode.OK) + { + Content = new StringContent(detailHtml, Encoding.UTF8, "text/html"), + }; + + response.Headers.ETag = etag; + response.Content.Headers.LastModified = lastModified; + return response; + }); + + var connector = new CertInConnectorPlugin().Create(provider); + + await connector.FetchAsync(provider, CancellationToken.None); + _timeProvider.Advance(TimeSpan.FromMinutes(1)); + await connector.ParseAsync(provider, CancellationToken.None); + await connector.MapAsync(provider, CancellationToken.None); + + var documentStore = provider.GetRequiredService(); + var document = await documentStore.FindBySourceAndUriAsync(CertInConnectorPlugin.SourceName, detailUri.ToString(), CancellationToken.None); + Assert.NotNull(document); + Assert.Equal(DocumentStatuses.Mapped, document!.Status); + Assert.Equal(etag.Tag, document.Etag); + + _handler.AddTextResponse(options.AlertsEndpoint, listingPayload, "application/json"); + _handler.AddResponse(detailUri, () => + { + var response = new HttpResponseMessage(HttpStatusCode.NotModified) + { + Content = new StringContent(string.Empty) + }; + response.Headers.ETag = etag; + return response; + }); + + await connector.FetchAsync(provider, CancellationToken.None); + await connector.ParseAsync(provider, CancellationToken.None); + await connector.MapAsync(provider, CancellationToken.None); + + document = await documentStore.FindBySourceAndUriAsync(CertInConnectorPlugin.SourceName, detailUri.ToString(), CancellationToken.None); + Assert.NotNull(document); + Assert.Equal(DocumentStatuses.Mapped, document!.Status); + + var stateRepository = provider.GetRequiredService(); + var state = await stateRepository.TryGetAsync(CertInConnectorPlugin.SourceName, CancellationToken.None); + Assert.NotNull(state); + Assert.True(state!.Cursor.TryGetValue("pendingDocuments", out var pendingDocs)); + Assert.Equal(0, pendingDocs.AsBsonArray.Count); + Assert.True(state.Cursor.TryGetValue("pendingMappings", out var pendingMappings)); + Assert.Equal(0, pendingMappings.AsBsonArray.Count); + } + + [Fact] + public async Task Fetch_DuplicateContentSkipsRequeue() + { + var options = new CertInOptions + { + AlertsEndpoint = new Uri("https://cert-in.example/api/alerts", UriKind.Absolute), + WindowSize = TimeSpan.FromDays(30), + WindowOverlap = TimeSpan.FromDays(7), + MaxPagesPerFetch = 1, + RequestDelay = TimeSpan.Zero, + }; + + await EnsureServiceProviderAsync(options); + var provider = _serviceProvider!; + _handler.Clear(); + + var listingPayload = ReadFixture("alerts-page1.json"); + var detailUri = new Uri("https://cert-in.example/advisory/CIAD-2024-0005"); + var detailHtml = ReadFixture("detail-CIAD-2024-0005.html"); + + _handler.AddTextResponse(options.AlertsEndpoint, listingPayload, "application/json"); + _handler.AddTextResponse(detailUri, detailHtml, "text/html"); + + var connector = new CertInConnectorPlugin().Create(provider); + + await connector.FetchAsync(provider, CancellationToken.None); + _timeProvider.Advance(TimeSpan.FromMinutes(1)); + await connector.ParseAsync(provider, CancellationToken.None); + await connector.MapAsync(provider, CancellationToken.None); + + var documentStore = provider.GetRequiredService(); + var document = await documentStore.FindBySourceAndUriAsync(CertInConnectorPlugin.SourceName, detailUri.ToString(), CancellationToken.None); + Assert.NotNull(document); + Assert.Equal(DocumentStatuses.Mapped, document!.Status); + + _handler.AddTextResponse(options.AlertsEndpoint, listingPayload, "application/json"); + _handler.AddTextResponse(detailUri, detailHtml, "text/html"); + + await connector.FetchAsync(provider, CancellationToken.None); + await connector.ParseAsync(provider, CancellationToken.None); + await connector.MapAsync(provider, CancellationToken.None); + + document = await documentStore.FindBySourceAndUriAsync(CertInConnectorPlugin.SourceName, detailUri.ToString(), CancellationToken.None); + Assert.NotNull(document); + Assert.Equal(DocumentStatuses.Mapped, document!.Status); + + var stateRepository = provider.GetRequiredService(); + var state = await stateRepository.TryGetAsync(CertInConnectorPlugin.SourceName, CancellationToken.None); + Assert.NotNull(state); + Assert.True(state!.Cursor.TryGetValue("pendingDocuments", out var pendingDocs)); + Assert.Equal(0, pendingDocs.AsBsonArray.Count); + Assert.True(state.Cursor.TryGetValue("pendingMappings", out var pendingMappings)); + Assert.Equal(0, pendingMappings.AsBsonArray.Count); + } + + private async Task EnsureServiceProviderAsync(CertInOptions template) + { + if (_serviceProvider is not null) + { + await ResetDatabaseAsync(); + return; + } + + await _fixture.Client.DropDatabaseAsync(_fixture.Database.DatabaseNamespace.DatabaseName); + + var services = new ServiceCollection(); + services.AddLogging(builder => builder.AddProvider(NullLoggerProvider.Instance)); + services.AddSingleton(_timeProvider); + services.AddSingleton(_handler); + + services.AddMongoStorage(options => + { + options.ConnectionString = _fixture.Runner.ConnectionString; + options.DatabaseName = _fixture.Database.DatabaseNamespace.DatabaseName; + options.CommandTimeout = TimeSpan.FromSeconds(5); + }); + + services.AddSourceCommon(); + services.AddCertInConnector(opts => + { + opts.AlertsEndpoint = template.AlertsEndpoint; + opts.WindowSize = template.WindowSize; + opts.WindowOverlap = template.WindowOverlap; + opts.MaxPagesPerFetch = template.MaxPagesPerFetch; + opts.RequestDelay = template.RequestDelay; + }); + + services.Configure(CertInOptions.HttpClientName, builderOptions => + { + builderOptions.HttpMessageHandlerBuilderActions.Add(builder => + { + builder.PrimaryHandler = _handler; + }); + }); + + _serviceProvider = services.BuildServiceProvider(); + var bootstrapper = _serviceProvider.GetRequiredService(); + await bootstrapper.InitializeAsync(CancellationToken.None); + } + + private Task ResetDatabaseAsync() + => _fixture.Client.DropDatabaseAsync(_fixture.Database.DatabaseNamespace.DatabaseName); + + private static string ReadFixture(string filename) + => File.ReadAllText(ResolveFixturePath(filename)); + + private static string ResolveFixturePath(string filename) + { + var baseDirectory = AppContext.BaseDirectory; + var primary = Path.Combine(baseDirectory, "Source", "CertIn", "Fixtures", filename); + if (File.Exists(primary) || filename.EndsWith(".actual.json", StringComparison.OrdinalIgnoreCase)) + { + return primary; + } + + return Path.Combine(baseDirectory, "CertIn", "Fixtures", filename); + } + + private static string NormalizeLineEndings(string value) + => value.Replace("\r\n", "\n", StringComparison.Ordinal); + + public Task InitializeAsync() => Task.CompletedTask; + + public async Task DisposeAsync() + { + if (_serviceProvider is IAsyncDisposable asyncDisposable) + { + await asyncDisposable.DisposeAsync(); + } + else + { + _serviceProvider?.Dispose(); + } + } +} diff --git a/src/StellaOps.Feedser.Source.CertIn.Tests/CertIn/Fixtures/alerts-page1.json b/src/StellaOps.Feedser.Source.CertIn.Tests/CertIn/Fixtures/alerts-page1.json new file mode 100644 index 00000000..dff5ff6b --- /dev/null +++ b/src/StellaOps.Feedser.Source.CertIn.Tests/CertIn/Fixtures/alerts-page1.json @@ -0,0 +1,9 @@ +[ + { + "advisoryId": "CIAD-2024-0005", + "title": "Multiple vulnerabilities in Example Gateway", + "publishedOn": "2024-04-15T10:00:00Z", + "detailUrl": "https://cert-in.example/advisory/CIAD-2024-0005", + "summary": "Example Gateway devices vulnerable to remote code execution (CVE-2024-9990)." + } +] diff --git a/src/StellaOps.Feedser.Source.CertIn.Tests/CertIn/Fixtures/detail-CIAD-2024-0005.html b/src/StellaOps.Feedser.Source.CertIn.Tests/CertIn/Fixtures/detail-CIAD-2024-0005.html new file mode 100644 index 00000000..9731f003 --- /dev/null +++ b/src/StellaOps.Feedser.Source.CertIn.Tests/CertIn/Fixtures/detail-CIAD-2024-0005.html @@ -0,0 +1,17 @@ + + + + + Multiple vulnerabilities in Example Gateway + + +
    +

    Multiple vulnerabilities in Example Gateway

    +

    Severity: High

    +

    Vendor: Example Gateway Technologies Pvt Ltd

    +

    Organisation: Partner Systems Inc.

    +

    CVE-2024-9990 and CVE-2024-9991 allow remote attackers to execute arbitrary commands.

    +

    Further information is available from the vendor bulletin.

    +
    + + diff --git a/src/StellaOps.Feedser.Source.CertIn.Tests/CertIn/Fixtures/expected-advisory.json b/src/StellaOps.Feedser.Source.CertIn.Tests/CertIn/Fixtures/expected-advisory.json new file mode 100644 index 00000000..e8f3dd5d --- /dev/null +++ b/src/StellaOps.Feedser.Source.CertIn.Tests/CertIn/Fixtures/expected-advisory.json @@ -0,0 +1,128 @@ +{ + "advisoryKey": "CIAD-2024-0005", + "affectedPackages": [ + { + "identifier": "Example Gateway Technologies Pvt Ltd Organisation: Partner Systems Inc. CVE-2024-9990 and CVE-2024-9991 allow remote attackers to execute arbitrary commands. Further information is available from the", + "platform": null, + "provenance": [ + { + "fieldMask": [], + "kind": "affected", + "recordedAt": "2024-04-20T00:01:00+00:00", + "source": "cert-in", + "value": "Example Gateway Technologies Pvt Ltd Organisation: Partner Systems Inc. CVE-2024-9990 and CVE-2024-9991 allow remote attackers to execute arbitrary commands. Further information is available from the" + } + ], + "statuses": [], + "type": "ics-vendor", + "versionRanges": [ + { + "fixedVersion": null, + "introducedVersion": null, + "lastAffectedVersion": null, + "primitives": { + "evr": null, + "hasVendorExtensions": true, + "nevra": null, + "semVer": null, + "vendorExtensions": { + "certin.vendor": "Example Gateway Technologies Pvt Ltd Organisation: Partner Systems Inc. CVE-2024-9990 and CVE-2024-9991 allow remote attackers to execute arbitrary commands. Further information is available from the " + } + }, + "provenance": { + "fieldMask": [], + "kind": "affected", + "recordedAt": "2024-04-20T00:01:00+00:00", + "source": "cert-in", + "value": "Example Gateway Technologies Pvt Ltd Organisation: Partner Systems Inc. CVE-2024-9990 and CVE-2024-9991 allow remote attackers to execute arbitrary commands. Further information is available from the" + }, + "rangeExpression": null, + "rangeKind": "vendor" + } + ] + } + ], + "aliases": [ + "CIAD-2024-0005", + "CVE-2024-9990", + "CVE-2024-9991" + ], + "cvssMetrics": [], + "exploitKnown": false, + "language": "en", + "modified": "2024-04-15T10:00:00+00:00", + "provenance": [ + { + "fieldMask": [], + "kind": "document", + "recordedAt": "2024-04-20T00:00:00+00:00", + "source": "cert-in", + "value": "https://cert-in.example/advisory/CIAD-2024-0005" + }, + { + "fieldMask": [], + "kind": "mapping", + "recordedAt": "2024-04-20T00:01:00+00:00", + "source": "cert-in", + "value": "CIAD-2024-0005" + } + ], + "published": "2024-04-15T10:00:00+00:00", + "references": [ + { + "kind": "advisory", + "provenance": { + "fieldMask": [], + "kind": "reference", + "recordedAt": "2024-04-20T00:01:00+00:00", + "source": "cert-in", + "value": "https://cert-in.example/advisory/CIAD-2024-0005" + }, + "sourceTag": "cert-in", + "summary": null, + "url": "https://cert-in.example/advisory/CIAD-2024-0005" + }, + { + "kind": "reference", + "provenance": { + "fieldMask": [], + "kind": "reference", + "recordedAt": "2024-04-20T00:01:00+00:00", + "source": "cert-in", + "value": "https://vendor.example.com/advisories/example-gateway-bulletin" + }, + "sourceTag": null, + "summary": null, + "url": "https://vendor.example.com/advisories/example-gateway-bulletin" + }, + { + "kind": "advisory", + "provenance": { + "fieldMask": [], + "kind": "reference", + "recordedAt": "2024-04-20T00:01:00+00:00", + "source": "cert-in", + "value": "https://www.cve.org/CVERecord?id=CVE-2024-9990" + }, + "sourceTag": "CVE-2024-9990", + "summary": null, + "url": "https://www.cve.org/CVERecord?id=CVE-2024-9990" + }, + { + "kind": "advisory", + "provenance": { + "fieldMask": [], + "kind": "reference", + "recordedAt": "2024-04-20T00:01:00+00:00", + "source": "cert-in", + "value": "https://www.cve.org/CVERecord?id=CVE-2024-9991" + }, + "sourceTag": "CVE-2024-9991", + "summary": null, + "url": "https://www.cve.org/CVERecord?id=CVE-2024-9991" + } + ], + "severity": "high", + "summary": "Example Gateway devices vulnerable to remote code execution (CVE-2024-9990).", + "title": "Multiple vulnerabilities in Example Gateway" +} \ No newline at end of file diff --git a/src/StellaOps.Feedser.Source.CertIn.Tests/StellaOps.Feedser.Source.CertIn.Tests.csproj b/src/StellaOps.Feedser.Source.CertIn.Tests/StellaOps.Feedser.Source.CertIn.Tests.csproj new file mode 100644 index 00000000..c7000b4f --- /dev/null +++ b/src/StellaOps.Feedser.Source.CertIn.Tests/StellaOps.Feedser.Source.CertIn.Tests.csproj @@ -0,0 +1,16 @@ + + + net10.0 + enable + enable + + + + + + + + + + + diff --git a/src/StellaOps.Feedser.Source.CertIn/AGENTS.md b/src/StellaOps.Feedser.Source.CertIn/AGENTS.md new file mode 100644 index 00000000..e8e4dc8f --- /dev/null +++ b/src/StellaOps.Feedser.Source.CertIn/AGENTS.md @@ -0,0 +1,28 @@ +# AGENTS +## Role +CERT-In national CERT connector; enrichment advisories for India; maps CVE lists, advisory text, mitigations, and references; non-authoritative for package ranges unless explicit evidence is present. +## Scope +- Discover and fetch advisories from the CERT-In portal; window by advisory code/date; follow detail pages. +- Validate HTML or JSON; extract title, summary, CVEs, affected vendor names, mitigations; map references; normalize dates and IDs. +- Persist raw docs and maintain source_state cursor; idempotent mapping. +## Participants +- Source.Common (HTTP, HTML parsing, normalization, validators). +- Storage.Mongo (document, dto, advisory, alias, reference, source_state). +- Models (canonical). +- Core/WebService (jobs: source:certin:fetch|parse|map). +- Merge engine treats CERT-In as enrichment (no override of PSIRT or OVAL without concrete ranges). +## Interfaces & contracts +- Aliases: advisory code if stable (scheme "CERT-IN") and CVE ids; if code is not stable, store as reference only. +- References typed: bulletin/advisory/vendor/mitigation; deduped. +- Affected omitted unless CERT-In publishes explicit version or fix details. +- Provenance: method=parser; value=advisory code or URL; recordedAt. +## In/Out of scope +In: enrichment, aliasing where stable, references, mitigation text. +Out: package range authority; scraping behind auth walls. +## Observability & security expectations +- Metrics: shared `feedser.source.http.*` counters/histograms from SourceDiagnostics tagged `feedser.source=certin` capture fetch volume, parse failures, and map enrich counts. +- Logs: advisory codes, CVE counts per advisory, timing; allowlist host; redact personal data if present. +## Tests +- Author and review coverage in `../StellaOps.Feedser.Source.CertIn.Tests`. +- Shared fixtures (e.g., `MongoIntegrationFixture`, `ConnectorTestHarness`) live in `../StellaOps.Feedser.Testing`. +- Keep fixtures deterministic; match new cases to real-world advisories or regression scenarios. diff --git a/src/StellaOps.Feedser.Source.CertIn/CertInConnector.cs b/src/StellaOps.Feedser.Source.CertIn/CertInConnector.cs new file mode 100644 index 00000000..e6e53534 --- /dev/null +++ b/src/StellaOps.Feedser.Source.CertIn/CertInConnector.cs @@ -0,0 +1,462 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text.Json; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Options; +using MongoDB.Bson; +using StellaOps.Feedser.Models; +using StellaOps.Feedser.Source.CertIn.Configuration; +using StellaOps.Feedser.Source.CertIn.Internal; +using StellaOps.Feedser.Source.Common; +using StellaOps.Feedser.Source.Common.Fetch; +using StellaOps.Feedser.Storage.Mongo; +using StellaOps.Feedser.Storage.Mongo.Advisories; +using StellaOps.Feedser.Storage.Mongo.Documents; +using StellaOps.Feedser.Storage.Mongo.Dtos; +using StellaOps.Plugin; + +namespace StellaOps.Feedser.Source.CertIn; + +public sealed class CertInConnector : IFeedConnector +{ + private static readonly JsonSerializerOptions SerializerOptions = new(JsonSerializerDefaults.General) + { + PropertyNamingPolicy = JsonNamingPolicy.CamelCase, + WriteIndented = false, + DefaultIgnoreCondition = System.Text.Json.Serialization.JsonIgnoreCondition.WhenWritingNull, + }; + + private readonly CertInClient _client; + private readonly SourceFetchService _fetchService; + private readonly RawDocumentStorage _rawDocumentStorage; + private readonly IDocumentStore _documentStore; + private readonly IDtoStore _dtoStore; + private readonly IAdvisoryStore _advisoryStore; + private readonly ISourceStateRepository _stateRepository; + private readonly CertInOptions _options; + private readonly TimeProvider _timeProvider; + private readonly ILogger _logger; + + public CertInConnector( + CertInClient client, + SourceFetchService fetchService, + RawDocumentStorage rawDocumentStorage, + IDocumentStore documentStore, + IDtoStore dtoStore, + IAdvisoryStore advisoryStore, + ISourceStateRepository stateRepository, + IOptions options, + TimeProvider? timeProvider, + ILogger logger) + { + _client = client ?? throw new ArgumentNullException(nameof(client)); + _fetchService = fetchService ?? throw new ArgumentNullException(nameof(fetchService)); + _rawDocumentStorage = rawDocumentStorage ?? throw new ArgumentNullException(nameof(rawDocumentStorage)); + _documentStore = documentStore ?? throw new ArgumentNullException(nameof(documentStore)); + _dtoStore = dtoStore ?? throw new ArgumentNullException(nameof(dtoStore)); + _advisoryStore = advisoryStore ?? throw new ArgumentNullException(nameof(advisoryStore)); + _stateRepository = stateRepository ?? throw new ArgumentNullException(nameof(stateRepository)); + _options = (options ?? throw new ArgumentNullException(nameof(options))).Value ?? throw new ArgumentNullException(nameof(options)); + _options.Validate(); + _timeProvider = timeProvider ?? TimeProvider.System; + _logger = logger ?? throw new ArgumentNullException(nameof(logger)); + } + + public string SourceName => CertInConnectorPlugin.SourceName; + + public async Task FetchAsync(IServiceProvider services, CancellationToken cancellationToken) + { + ArgumentNullException.ThrowIfNull(services); + + var cursor = await GetCursorAsync(cancellationToken).ConfigureAwait(false); + var now = _timeProvider.GetUtcNow(); + var windowStart = cursor.LastPublished.HasValue + ? cursor.LastPublished.Value - _options.WindowOverlap + : now - _options.WindowSize; + + var pendingDocuments = cursor.PendingDocuments.ToHashSet(); + var maxPublished = cursor.LastPublished ?? DateTimeOffset.MinValue; + + for (var page = 1; page <= _options.MaxPagesPerFetch; page++) + { + IReadOnlyList listings; + try + { + listings = await _client.GetListingsAsync(page, cancellationToken).ConfigureAwait(false); + } + catch (Exception ex) + { + _logger.LogError(ex, "CERT-In listings fetch failed for page {Page}", page); + await _stateRepository.MarkFailureAsync(SourceName, now, TimeSpan.FromMinutes(5), ex.Message, cancellationToken).ConfigureAwait(false); + throw; + } + if (listings.Count == 0) + { + break; + } + + foreach (var listing in listings.OrderByDescending(static item => item.Published)) + { + if (listing.Published < windowStart) + { + page = _options.MaxPagesPerFetch + 1; + break; + } + + var metadata = new Dictionary(StringComparer.Ordinal) + { + ["certin.advisoryId"] = listing.AdvisoryId, + ["certin.title"] = listing.Title, + ["certin.link"] = listing.DetailUri.ToString(), + ["certin.published"] = listing.Published.ToString("O") + }; + + if (!string.IsNullOrWhiteSpace(listing.Summary)) + { + metadata["certin.summary"] = listing.Summary!; + } + + var existing = await _documentStore.FindBySourceAndUriAsync(SourceName, listing.DetailUri.ToString(), cancellationToken).ConfigureAwait(false); + + SourceFetchResult result; + try + { + result = await _fetchService.FetchAsync( + new SourceFetchRequest(CertInOptions.HttpClientName, SourceName, listing.DetailUri) + { + Metadata = metadata, + ETag = existing?.Etag, + LastModified = existing?.LastModified, + AcceptHeaders = new[] { "text/html", "application/xhtml+xml", "text/plain;q=0.5" }, + }, + cancellationToken).ConfigureAwait(false); + } + catch (Exception ex) + { + _logger.LogError(ex, "CERT-In fetch failed for {Uri}", listing.DetailUri); + await _stateRepository.MarkFailureAsync(SourceName, _timeProvider.GetUtcNow(), TimeSpan.FromMinutes(3), ex.Message, cancellationToken).ConfigureAwait(false); + throw; + } + + if (!result.IsSuccess || result.Document is null) + { + continue; + } + + if (existing is not null + && string.Equals(existing.Sha256, result.Document.Sha256, StringComparison.OrdinalIgnoreCase) + && string.Equals(existing.Status, DocumentStatuses.Mapped, StringComparison.Ordinal)) + { + await _documentStore.UpdateStatusAsync(result.Document.Id, existing.Status, cancellationToken).ConfigureAwait(false); + continue; + } + + pendingDocuments.Add(result.Document.Id); + if (listing.Published > maxPublished) + { + maxPublished = listing.Published; + } + + if (_options.RequestDelay > TimeSpan.Zero) + { + await Task.Delay(_options.RequestDelay, cancellationToken).ConfigureAwait(false); + } + } + } + + var updatedCursor = cursor + .WithPendingDocuments(pendingDocuments) + .WithLastPublished(maxPublished == DateTimeOffset.MinValue ? cursor.LastPublished : maxPublished); + + await UpdateCursorAsync(updatedCursor, cancellationToken).ConfigureAwait(false); + } + + public async Task ParseAsync(IServiceProvider services, CancellationToken cancellationToken) + { + ArgumentNullException.ThrowIfNull(services); + + var cursor = await GetCursorAsync(cancellationToken).ConfigureAwait(false); + if (cursor.PendingDocuments.Count == 0) + { + return; + } + + var remainingDocuments = cursor.PendingDocuments.ToList(); + var pendingMappings = cursor.PendingMappings.ToList(); + + foreach (var documentId in cursor.PendingDocuments) + { + cancellationToken.ThrowIfCancellationRequested(); + + var document = await _documentStore.FindAsync(documentId, cancellationToken).ConfigureAwait(false); + if (document is null) + { + remainingDocuments.Remove(documentId); + continue; + } + + if (!document.GridFsId.HasValue) + { + _logger.LogWarning("CERT-In document {DocumentId} missing GridFS payload", document.Id); + await _documentStore.UpdateStatusAsync(document.Id, DocumentStatuses.Failed, cancellationToken).ConfigureAwait(false); + remainingDocuments.Remove(documentId); + continue; + } + + if (!TryDeserializeListing(document.Metadata, out var listing)) + { + _logger.LogWarning("CERT-In metadata missing for {DocumentId}", document.Id); + await _documentStore.UpdateStatusAsync(document.Id, DocumentStatuses.Failed, cancellationToken).ConfigureAwait(false); + remainingDocuments.Remove(documentId); + continue; + } + + byte[] rawBytes; + try + { + rawBytes = await _rawDocumentStorage.DownloadAsync(document.GridFsId.Value, cancellationToken).ConfigureAwait(false); + } + catch (Exception ex) + { + _logger.LogError(ex, "Failed to download raw CERT-In document {DocumentId}", document.Id); + throw; + } + + var dto = CertInDetailParser.Parse(listing, rawBytes); + var payload = BsonDocument.Parse(JsonSerializer.Serialize(dto, SerializerOptions)); + var dtoRecord = new DtoRecord(Guid.NewGuid(), document.Id, SourceName, "certin.v1", payload, _timeProvider.GetUtcNow()); + + await _dtoStore.UpsertAsync(dtoRecord, cancellationToken).ConfigureAwait(false); + await _documentStore.UpdateStatusAsync(document.Id, DocumentStatuses.PendingMap, cancellationToken).ConfigureAwait(false); + + remainingDocuments.Remove(documentId); + if (!pendingMappings.Contains(documentId)) + { + pendingMappings.Add(documentId); + } + } + + var updatedCursor = cursor + .WithPendingDocuments(remainingDocuments) + .WithPendingMappings(pendingMappings); + + await UpdateCursorAsync(updatedCursor, cancellationToken).ConfigureAwait(false); + } + + public async Task MapAsync(IServiceProvider services, CancellationToken cancellationToken) + { + ArgumentNullException.ThrowIfNull(services); + + var cursor = await GetCursorAsync(cancellationToken).ConfigureAwait(false); + if (cursor.PendingMappings.Count == 0) + { + return; + } + + var pendingMappings = cursor.PendingMappings.ToList(); + + foreach (var documentId in cursor.PendingMappings) + { + cancellationToken.ThrowIfCancellationRequested(); + + var dtoRecord = await _dtoStore.FindByDocumentIdAsync(documentId, cancellationToken).ConfigureAwait(false); + var document = await _documentStore.FindAsync(documentId, cancellationToken).ConfigureAwait(false); + + if (dtoRecord is null || document is null) + { + pendingMappings.Remove(documentId); + continue; + } + + var dtoJson = dtoRecord.Payload.ToJson(new MongoDB.Bson.IO.JsonWriterSettings + { + OutputMode = MongoDB.Bson.IO.JsonOutputMode.RelaxedExtendedJson, + }); + + CertInAdvisoryDto dto; + try + { + dto = JsonSerializer.Deserialize(dtoJson, SerializerOptions) + ?? throw new InvalidOperationException("Deserialized CERT-In DTO is null."); + } + catch (Exception ex) + { + _logger.LogError(ex, "Failed to deserialize CERT-In DTO for {DocumentId}", document.Id); + await _documentStore.UpdateStatusAsync(document.Id, DocumentStatuses.Failed, cancellationToken).ConfigureAwait(false); + pendingMappings.Remove(documentId); + continue; + } + + var advisory = MapAdvisory(dto, document, dtoRecord); + await _advisoryStore.UpsertAsync(advisory, cancellationToken).ConfigureAwait(false); + await _documentStore.UpdateStatusAsync(document.Id, DocumentStatuses.Mapped, cancellationToken).ConfigureAwait(false); + + pendingMappings.Remove(documentId); + } + + var updatedCursor = cursor.WithPendingMappings(pendingMappings); + await UpdateCursorAsync(updatedCursor, cancellationToken).ConfigureAwait(false); + } + + private Advisory MapAdvisory(CertInAdvisoryDto dto, DocumentRecord document, DtoRecord dtoRecord) + { + var fetchProvenance = new AdvisoryProvenance(SourceName, "document", document.Uri, document.FetchedAt); + var mappingProvenance = new AdvisoryProvenance(SourceName, "mapping", dto.AdvisoryId, dtoRecord.ValidatedAt); + + var aliases = new HashSet(StringComparer.OrdinalIgnoreCase) + { + dto.AdvisoryId, + }; + foreach (var cve in dto.CveIds) + { + aliases.Add(cve); + } + + var references = new List(); + try + { + references.Add(new AdvisoryReference( + dto.Link, + "advisory", + "cert-in", + null, + new AdvisoryProvenance(SourceName, "reference", dto.Link, dtoRecord.ValidatedAt))); + } + catch (ArgumentException) + { + _logger.LogWarning("Invalid CERT-In link {Link} for advisory {AdvisoryId}", dto.Link, dto.AdvisoryId); + } + + foreach (var cve in dto.CveIds) + { + var url = $"https://www.cve.org/CVERecord?id={cve}"; + try + { + references.Add(new AdvisoryReference( + url, + "advisory", + cve, + null, + new AdvisoryProvenance(SourceName, "reference", url, dtoRecord.ValidatedAt))); + } + catch (ArgumentException) + { + // ignore invalid urls + } + } + + foreach (var link in dto.ReferenceLinks) + { + try + { + references.Add(new AdvisoryReference( + link, + "reference", + null, + null, + new AdvisoryProvenance(SourceName, "reference", link, dtoRecord.ValidatedAt))); + } + catch (ArgumentException) + { + // ignore invalid urls + } + } + + var affectedPackages = dto.VendorNames.Select(vendor => + { + var provenance = new AdvisoryProvenance(SourceName, "affected", vendor, dtoRecord.ValidatedAt); + var primitives = new RangePrimitives( + null, + null, + null, + new Dictionary(StringComparer.OrdinalIgnoreCase) + { + ["certin.vendor"] = vendor + }); + + var ranges = new[] + { + new AffectedVersionRange( + rangeKind: "vendor", + introducedVersion: null, + fixedVersion: null, + lastAffectedVersion: null, + rangeExpression: null, + provenance: provenance, + primitives: primitives) + }; + + return new AffectedPackage( + AffectedPackageTypes.IcsVendor, + vendor, + platform: null, + versionRanges: ranges, + statuses: Array.Empty(), + provenance: new[] { provenance }); + }) + .ToArray(); + + return new Advisory( + dto.AdvisoryId, + dto.Title, + dto.Summary ?? dto.Content, + language: "en", + published: dto.Published, + modified: dto.Published, + severity: dto.Severity, + exploitKnown: false, + aliases: aliases, + references: references, + affectedPackages: affectedPackages, + cvssMetrics: Array.Empty(), + provenance: new[] { fetchProvenance, mappingProvenance }); + } + + private async Task GetCursorAsync(CancellationToken cancellationToken) + { + var state = await _stateRepository.TryGetAsync(SourceName, cancellationToken).ConfigureAwait(false); + return state is null ? CertInCursor.Empty : CertInCursor.FromBson(state.Cursor); + } + + private Task UpdateCursorAsync(CertInCursor cursor, CancellationToken cancellationToken) + { + return _stateRepository.UpdateCursorAsync(SourceName, cursor.ToBsonDocument(), _timeProvider.GetUtcNow(), cancellationToken); + } + + private static bool TryDeserializeListing(IReadOnlyDictionary? metadata, out CertInListingItem listing) + { + listing = null!; + if (metadata is null) + { + return false; + } + + if (!metadata.TryGetValue("certin.advisoryId", out var advisoryId)) + { + return false; + } + + if (!metadata.TryGetValue("certin.title", out var title)) + { + return false; + } + + if (!metadata.TryGetValue("certin.link", out var link) || !Uri.TryCreate(link, UriKind.Absolute, out var detailUri)) + { + return false; + } + + if (!metadata.TryGetValue("certin.published", out var publishedText) || !DateTimeOffset.TryParse(publishedText, out var published)) + { + return false; + } + + metadata.TryGetValue("certin.summary", out var summary); + + listing = new CertInListingItem(advisoryId, title, detailUri, published.ToUniversalTime(), summary); + return true; + } +} diff --git a/src/StellaOps.Feedser.Source.CertIn/CertInConnectorPlugin.cs b/src/StellaOps.Feedser.Source.CertIn/CertInConnectorPlugin.cs new file mode 100644 index 00000000..a25ae81d --- /dev/null +++ b/src/StellaOps.Feedser.Source.CertIn/CertInConnectorPlugin.cs @@ -0,0 +1,19 @@ +using Microsoft.Extensions.DependencyInjection; +using StellaOps.Plugin; + +namespace StellaOps.Feedser.Source.CertIn; + +public sealed class CertInConnectorPlugin : IConnectorPlugin +{ + public const string SourceName = "cert-in"; + + public string Name => SourceName; + + public bool IsAvailable(IServiceProvider services) => services is not null; + + public IFeedConnector Create(IServiceProvider services) + { + ArgumentNullException.ThrowIfNull(services); + return ActivatorUtilities.CreateInstance(services); + } +} diff --git a/src/StellaOps.Feedser.Source.CertIn/CertInDependencyInjectionRoutine.cs b/src/StellaOps.Feedser.Source.CertIn/CertInDependencyInjectionRoutine.cs new file mode 100644 index 00000000..662d1411 --- /dev/null +++ b/src/StellaOps.Feedser.Source.CertIn/CertInDependencyInjectionRoutine.cs @@ -0,0 +1,54 @@ +using System; +using Microsoft.Extensions.Configuration; +using Microsoft.Extensions.DependencyInjection; +using StellaOps.DependencyInjection; +using StellaOps.Feedser.Core.Jobs; +using StellaOps.Feedser.Source.CertIn.Configuration; + +namespace StellaOps.Feedser.Source.CertIn; + +public sealed class CertInDependencyInjectionRoutine : IDependencyInjectionRoutine +{ + private const string ConfigurationSection = "feedser:sources:cert-in"; + + public IServiceCollection Register(IServiceCollection services, IConfiguration configuration) + { + ArgumentNullException.ThrowIfNull(services); + ArgumentNullException.ThrowIfNull(configuration); + + services.AddCertInConnector(options => + { + configuration.GetSection(ConfigurationSection).Bind(options); + options.Validate(); + }); + + services.AddTransient(); + services.AddTransient(); + services.AddTransient(); + + services.PostConfigure(options => + { + EnsureJob(options, CertInJobKinds.Fetch, typeof(CertInFetchJob)); + EnsureJob(options, CertInJobKinds.Parse, typeof(CertInParseJob)); + EnsureJob(options, CertInJobKinds.Map, typeof(CertInMapJob)); + }); + + return services; + } + + private static void EnsureJob(JobSchedulerOptions options, string kind, Type jobType) + { + if (options.Definitions.ContainsKey(kind)) + { + return; + } + + options.Definitions[kind] = new JobDefinition( + kind, + jobType, + options.DefaultTimeout, + options.DefaultLeaseDuration, + CronExpression: null, + Enabled: true); + } +} diff --git a/src/StellaOps.Feedser.Source.CertIn/CertInServiceCollectionExtensions.cs b/src/StellaOps.Feedser.Source.CertIn/CertInServiceCollectionExtensions.cs new file mode 100644 index 00000000..8b1feb47 --- /dev/null +++ b/src/StellaOps.Feedser.Source.CertIn/CertInServiceCollectionExtensions.cs @@ -0,0 +1,37 @@ +using System; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Options; +using StellaOps.Feedser.Source.CertIn.Configuration; +using StellaOps.Feedser.Source.CertIn.Internal; +using StellaOps.Feedser.Source.Common.Http; + +namespace StellaOps.Feedser.Source.CertIn; + +public static class CertInServiceCollectionExtensions +{ + public static IServiceCollection AddCertInConnector(this IServiceCollection services, Action configure) + { + ArgumentNullException.ThrowIfNull(services); + ArgumentNullException.ThrowIfNull(configure); + + services.AddOptions() + .Configure(configure) + .PostConfigure(static opts => opts.Validate()); + + services.AddSourceHttpClient(CertInOptions.HttpClientName, (sp, clientOptions) => + { + var options = sp.GetRequiredService>().Value; + clientOptions.BaseAddress = options.AlertsEndpoint; + clientOptions.Timeout = TimeSpan.FromSeconds(30); + clientOptions.UserAgent = "StellaOps.Feedser.CertIn/1.0"; + clientOptions.AllowedHosts.Clear(); + clientOptions.AllowedHosts.Add(options.AlertsEndpoint.Host); + clientOptions.DefaultRequestHeaders["Accept"] = "application/json"; + }); + + services.AddTransient(); + services.AddTransient(); + + return services; + } +} diff --git a/src/StellaOps.Feedser.Source.CertIn/Configuration/CertInOptions.cs b/src/StellaOps.Feedser.Source.CertIn/Configuration/CertInOptions.cs new file mode 100644 index 00000000..88a69095 --- /dev/null +++ b/src/StellaOps.Feedser.Source.CertIn/Configuration/CertInOptions.cs @@ -0,0 +1,68 @@ +using System; +using System.Diagnostics.CodeAnalysis; + +namespace StellaOps.Feedser.Source.CertIn.Configuration; + +public sealed class CertInOptions +{ + public static string HttpClientName => "source.certin"; + + /// + /// Endpoint returning a paginated list of recent advisories. + /// + public Uri AlertsEndpoint { get; set; } = new("https://www.cert-in.org.in/api/alerts", UriKind.Absolute); + + /// + /// Size of the rolling fetch window. + /// + public TimeSpan WindowSize { get; set; } = TimeSpan.FromDays(30); + + /// + /// Overlap applied to subsequent windows. + /// + public TimeSpan WindowOverlap { get; set; } = TimeSpan.FromDays(2); + + /// + /// Maximum pages fetched per cycle. + /// + public int MaxPagesPerFetch { get; set; } = 5; + + /// + /// Delay between successive HTTP requests. + /// + public TimeSpan RequestDelay { get; set; } = TimeSpan.FromMilliseconds(500); + + [MemberNotNull(nameof(AlertsEndpoint))] + public void Validate() + { + if (AlertsEndpoint is null || !AlertsEndpoint.IsAbsoluteUri) + { + throw new InvalidOperationException("AlertsEndpoint must be an absolute URI."); + } + + if (WindowSize <= TimeSpan.Zero) + { + throw new InvalidOperationException("WindowSize must be greater than zero."); + } + + if (WindowOverlap < TimeSpan.Zero) + { + throw new InvalidOperationException("WindowOverlap cannot be negative."); + } + + if (WindowOverlap >= WindowSize) + { + throw new InvalidOperationException("WindowOverlap must be smaller than WindowSize."); + } + + if (MaxPagesPerFetch <= 0) + { + throw new InvalidOperationException("MaxPagesPerFetch must be positive."); + } + + if (RequestDelay < TimeSpan.Zero) + { + throw new InvalidOperationException("RequestDelay cannot be negative."); + } + } +} diff --git a/src/StellaOps.Feedser.Source.CertIn/Internal/CertInAdvisoryDto.cs b/src/StellaOps.Feedser.Source.CertIn/Internal/CertInAdvisoryDto.cs new file mode 100644 index 00000000..3bf3cbbf --- /dev/null +++ b/src/StellaOps.Feedser.Source.CertIn/Internal/CertInAdvisoryDto.cs @@ -0,0 +1,16 @@ +using System; +using System.Collections.Immutable; + +namespace StellaOps.Feedser.Source.CertIn.Internal; + +internal sealed record CertInAdvisoryDto( + string AdvisoryId, + string Title, + string Link, + DateTimeOffset Published, + string? Summary, + string Content, + string? Severity, + ImmutableArray CveIds, + ImmutableArray VendorNames, + ImmutableArray ReferenceLinks); diff --git a/src/StellaOps.Feedser.Source.CertIn/Internal/CertInClient.cs b/src/StellaOps.Feedser.Source.CertIn/Internal/CertInClient.cs new file mode 100644 index 00000000..6e3119f5 --- /dev/null +++ b/src/StellaOps.Feedser.Source.CertIn/Internal/CertInClient.cs @@ -0,0 +1,129 @@ +using System; +using System.Collections.Generic; +using System.Globalization; +using System.IO; +using System.Linq; +using System.Net.Http; +using System.Text.Json; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Options; +using StellaOps.Feedser.Source.CertIn.Configuration; + +namespace StellaOps.Feedser.Source.CertIn.Internal; + +public sealed class CertInClient +{ + private readonly IHttpClientFactory _httpClientFactory; + private readonly CertInOptions _options; + private readonly ILogger _logger; + + public CertInClient(IHttpClientFactory httpClientFactory, IOptions options, ILogger logger) + { + _httpClientFactory = httpClientFactory ?? throw new ArgumentNullException(nameof(httpClientFactory)); + _options = (options ?? throw new ArgumentNullException(nameof(options))).Value ?? throw new ArgumentNullException(nameof(options)); + _options.Validate(); + _logger = logger ?? throw new ArgumentNullException(nameof(logger)); + } + + public async Task> GetListingsAsync(int page, CancellationToken cancellationToken) + { + var client = _httpClientFactory.CreateClient(CertInOptions.HttpClientName); + var requestUri = BuildPageUri(_options.AlertsEndpoint, page); + + using var response = await client.GetAsync(requestUri, cancellationToken).ConfigureAwait(false); + response.EnsureSuccessStatusCode(); + + await using var stream = await response.Content.ReadAsStreamAsync(cancellationToken).ConfigureAwait(false); + using var document = await JsonDocument.ParseAsync(stream, cancellationToken: cancellationToken).ConfigureAwait(false); + + var root = document.RootElement; + if (root.ValueKind != JsonValueKind.Array) + { + _logger.LogWarning("Unexpected CERT-In alert payload shape for {Uri}", requestUri); + return Array.Empty(); + } + + var items = new List(capacity: root.GetArrayLength()); + foreach (var element in root.EnumerateArray()) + { + if (!TryParseListing(element, out var item)) + { + continue; + } + + items.Add(item); + } + + return items; + } + + private static bool TryParseListing(JsonElement element, out CertInListingItem item) + { + item = null!; + + if (!element.TryGetProperty("advisoryId", out var idElement) || idElement.ValueKind != JsonValueKind.String) + { + return false; + } + + var advisoryId = idElement.GetString(); + if (string.IsNullOrWhiteSpace(advisoryId)) + { + return false; + } + + var title = element.TryGetProperty("title", out var titleElement) && titleElement.ValueKind == JsonValueKind.String + ? titleElement.GetString() + : advisoryId; + + if (!element.TryGetProperty("detailUrl", out var linkElement) || linkElement.ValueKind != JsonValueKind.String) + { + return false; + } + + if (!Uri.TryCreate(linkElement.GetString(), UriKind.Absolute, out var detailUri)) + { + return false; + } + + DateTimeOffset published; + if (element.TryGetProperty("publishedOn", out var publishedElement) && publishedElement.ValueKind == JsonValueKind.String) + { + if (!DateTimeOffset.TryParse(publishedElement.GetString(), CultureInfo.InvariantCulture, DateTimeStyles.AssumeUniversal, out published)) + { + return false; + } + } + else + { + return false; + } + + string? summary = null; + if (element.TryGetProperty("summary", out var summaryElement) && summaryElement.ValueKind == JsonValueKind.String) + { + summary = summaryElement.GetString(); + } + + item = new CertInListingItem(advisoryId.Trim(), title?.Trim() ?? advisoryId.Trim(), detailUri, published.ToUniversalTime(), summary?.Trim()); + return true; + } + + private static Uri BuildPageUri(Uri baseUri, int page) + { + if (page <= 1) + { + return baseUri; + } + + var builder = new UriBuilder(baseUri); + var trimmed = builder.Query.TrimStart('?'); + var pageSegment = $"page={page.ToString(CultureInfo.InvariantCulture)}"; + builder.Query = string.IsNullOrEmpty(trimmed) + ? pageSegment + : $"{trimmed}&{pageSegment}"; + return builder.Uri; + } +} diff --git a/src/StellaOps.Feedser.Source.CertIn/Internal/CertInCursor.cs b/src/StellaOps.Feedser.Source.CertIn/Internal/CertInCursor.cs new file mode 100644 index 00000000..835a7194 --- /dev/null +++ b/src/StellaOps.Feedser.Source.CertIn/Internal/CertInCursor.cs @@ -0,0 +1,88 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using MongoDB.Bson; + +namespace StellaOps.Feedser.Source.CertIn.Internal; + +internal sealed record CertInCursor( + DateTimeOffset? LastPublished, + IReadOnlyCollection PendingDocuments, + IReadOnlyCollection PendingMappings) +{ + public static CertInCursor Empty { get; } = new(null, Array.Empty(), Array.Empty()); + + public BsonDocument ToBsonDocument() + { + var document = new BsonDocument + { + ["pendingDocuments"] = new BsonArray(PendingDocuments.Select(id => id.ToString())), + ["pendingMappings"] = new BsonArray(PendingMappings.Select(id => id.ToString())), + }; + + if (LastPublished.HasValue) + { + document["lastPublished"] = LastPublished.Value.UtcDateTime; + } + + return document; + } + + public static CertInCursor FromBson(BsonDocument? document) + { + if (document is null || document.ElementCount == 0) + { + return Empty; + } + + var lastPublished = document.TryGetValue("lastPublished", out var dateValue) + ? ParseDate(dateValue) + : null; + + return new CertInCursor( + lastPublished, + ReadGuidArray(document, "pendingDocuments"), + ReadGuidArray(document, "pendingMappings")); + } + + public CertInCursor WithLastPublished(DateTimeOffset? timestamp) + => this with { LastPublished = timestamp }; + + public CertInCursor WithPendingDocuments(IEnumerable ids) + => this with { PendingDocuments = ids?.Distinct().ToArray() ?? Array.Empty() }; + + public CertInCursor WithPendingMappings(IEnumerable ids) + => this with { PendingMappings = ids?.Distinct().ToArray() ?? Array.Empty() }; + + private static DateTimeOffset? ParseDate(BsonValue value) + => value.BsonType switch + { + BsonType.DateTime => DateTime.SpecifyKind(value.ToUniversalTime(), DateTimeKind.Utc), + BsonType.String when DateTimeOffset.TryParse(value.AsString, out var parsed) => parsed.ToUniversalTime(), + _ => null, + }; + + private static IReadOnlyCollection ReadGuidArray(BsonDocument document, string field) + { + if (!document.TryGetValue(field, out var value) || value is not BsonArray array) + { + return Array.Empty(); + } + + var results = new List(array.Count); + foreach (var element in array) + { + if (element is null) + { + continue; + } + + if (Guid.TryParse(element.ToString(), out var guid)) + { + results.Add(guid); + } + } + + return results; + } +} diff --git a/src/StellaOps.Feedser.Source.CertIn/Internal/CertInDetailParser.cs b/src/StellaOps.Feedser.Source.CertIn/Internal/CertInDetailParser.cs new file mode 100644 index 00000000..24da33ee --- /dev/null +++ b/src/StellaOps.Feedser.Source.CertIn/Internal/CertInDetailParser.cs @@ -0,0 +1,187 @@ +using System; +using System.Collections.Generic; +using System.Collections.Immutable; +using System.Linq; +using System.Text; +using System.Text.RegularExpressions; + +namespace StellaOps.Feedser.Source.CertIn.Internal; + +internal static class CertInDetailParser +{ + private static readonly Regex CveRegex = new("CVE-\\d{4}-\\d+", RegexOptions.IgnoreCase | RegexOptions.Compiled); + private static readonly Regex SeverityRegex = new("Severity\\s*[:\\-]\\s*(?[A-Za-z ]{1,32})", RegexOptions.IgnoreCase | RegexOptions.Compiled); + private static readonly Regex VendorRegex = new("(?:Vendor|Organisation|Organization|Company)\\s*[:\\-]\\s*(?[^\\n\\r]+)", RegexOptions.IgnoreCase | RegexOptions.Compiled); + private static readonly Regex LinkRegex = new("href=\"(https?://[^\"]+)\"", RegexOptions.IgnoreCase | RegexOptions.Compiled); + + public static CertInAdvisoryDto Parse(CertInListingItem listing, byte[] rawHtml) + { + ArgumentNullException.ThrowIfNull(listing); + + var html = Encoding.UTF8.GetString(rawHtml); + var content = HtmlToPlainText(html); + var summary = listing.Summary ?? ExtractSummary(content); + var severity = ExtractSeverity(content); + var cves = ExtractCves(listing.Title, summary, content); + var vendors = ExtractVendors(summary, content); + var references = ExtractLinks(html); + + return new CertInAdvisoryDto( + listing.AdvisoryId, + listing.Title, + listing.DetailUri.ToString(), + listing.Published, + summary, + content, + severity, + cves, + vendors, + references); + } + + private static string HtmlToPlainText(string html) + { + if (string.IsNullOrWhiteSpace(html)) + { + return string.Empty; + } + + var withoutScripts = Regex.Replace(html, "", string.Empty, RegexOptions.IgnoreCase); + var withoutStyles = Regex.Replace(withoutScripts, "", string.Empty, RegexOptions.IgnoreCase); + var withoutComments = Regex.Replace(withoutStyles, "", string.Empty, RegexOptions.Singleline); + var withoutTags = Regex.Replace(withoutComments, "<[^>]+>", " "); + var decoded = System.Net.WebUtility.HtmlDecode(withoutTags); + return string.IsNullOrWhiteSpace(decoded) + ? string.Empty + : Regex.Replace(decoded, "\\s+", " ").Trim(); + } + + private static string? ExtractSummary(string content) + { + if (string.IsNullOrWhiteSpace(content)) + { + return null; + } + + var sentenceTerminators = new[] { ".", "!", "?" }; + foreach (var terminator in sentenceTerminators) + { + var index = content.IndexOf(terminator, StringComparison.Ordinal); + if (index > 0) + { + return content[..(index + terminator.Length)].Trim(); + } + } + + return content.Length > 280 ? content[..280].Trim() : content; + } + + private static string? ExtractSeverity(string content) + { + var match = SeverityRegex.Match(content); + if (match.Success) + { + return match.Groups["value"].Value.Trim().ToLowerInvariant(); + } + + return null; + } + + private static ImmutableArray ExtractCves(string title, string? summary, string content) + { + var set = new HashSet(StringComparer.OrdinalIgnoreCase); + + void Capture(string? text) + { + if (string.IsNullOrWhiteSpace(text)) + { + return; + } + + foreach (Match match in CveRegex.Matches(text)) + { + if (match.Success) + { + set.Add(match.Value.ToUpperInvariant()); + } + } + } + + Capture(title); + Capture(summary); + Capture(content); + + return set.OrderBy(static value => value, StringComparer.Ordinal).ToImmutableArray(); + } + + private static ImmutableArray ExtractVendors(string? summary, string content) + { + var vendors = new HashSet(StringComparer.OrdinalIgnoreCase); + + void Add(string? value) + { + if (string.IsNullOrWhiteSpace(value)) + { + return; + } + + var cleaned = value + .Replace("’", "'", StringComparison.Ordinal) + .Trim(); + + if (cleaned.Length > 200) + { + cleaned = cleaned[..200]; + } + + if (!string.IsNullOrWhiteSpace(cleaned)) + { + vendors.Add(cleaned); + } + } + + if (!string.IsNullOrWhiteSpace(summary)) + { + foreach (Match match in VendorRegex.Matches(summary)) + { + Add(match.Groups["value"].Value); + } + } + + foreach (Match match in VendorRegex.Matches(content)) + { + Add(match.Groups["value"].Value); + } + + if (vendors.Count == 0 && !string.IsNullOrWhiteSpace(summary)) + { + var fallback = summary.Split('.', 2, StringSplitOptions.TrimEntries | StringSplitOptions.RemoveEmptyEntries).FirstOrDefault(); + Add(fallback); + } + + return vendors.Count == 0 + ? ImmutableArray.Empty + : vendors.OrderBy(static value => value, StringComparer.Ordinal).ToImmutableArray(); + } + + private static ImmutableArray ExtractLinks(string html) + { + if (string.IsNullOrWhiteSpace(html)) + { + return ImmutableArray.Empty; + } + + var links = new HashSet(StringComparer.OrdinalIgnoreCase); + foreach (Match match in LinkRegex.Matches(html)) + { + if (match.Success) + { + links.Add(match.Groups[1].Value); + } + } + + return links.Count == 0 + ? ImmutableArray.Empty + : links.OrderBy(static value => value, StringComparer.Ordinal).ToImmutableArray(); + } +} diff --git a/src/StellaOps.Feedser.Source.CertIn/Internal/CertInListingItem.cs b/src/StellaOps.Feedser.Source.CertIn/Internal/CertInListingItem.cs new file mode 100644 index 00000000..3da8a3f3 --- /dev/null +++ b/src/StellaOps.Feedser.Source.CertIn/Internal/CertInListingItem.cs @@ -0,0 +1,10 @@ +using System; + +namespace StellaOps.Feedser.Source.CertIn.Internal; + +public sealed record CertInListingItem( + string AdvisoryId, + string Title, + Uri DetailUri, + DateTimeOffset Published, + string? Summary); diff --git a/src/StellaOps.Feedser.Source.CertIn/Jobs.cs b/src/StellaOps.Feedser.Source.CertIn/Jobs.cs new file mode 100644 index 00000000..01b35089 --- /dev/null +++ b/src/StellaOps.Feedser.Source.CertIn/Jobs.cs @@ -0,0 +1,46 @@ +using System; +using System.Threading; +using System.Threading.Tasks; +using StellaOps.Feedser.Core.Jobs; + +namespace StellaOps.Feedser.Source.CertIn; + +internal static class CertInJobKinds +{ + public const string Fetch = "source:cert-in:fetch"; + public const string Parse = "source:cert-in:parse"; + public const string Map = "source:cert-in:map"; +} + +internal sealed class CertInFetchJob : IJob +{ + private readonly CertInConnector _connector; + + public CertInFetchJob(CertInConnector connector) + => _connector = connector ?? throw new ArgumentNullException(nameof(connector)); + + public Task ExecuteAsync(JobExecutionContext context, CancellationToken cancellationToken) + => _connector.FetchAsync(context.Services, cancellationToken); +} + +internal sealed class CertInParseJob : IJob +{ + private readonly CertInConnector _connector; + + public CertInParseJob(CertInConnector connector) + => _connector = connector ?? throw new ArgumentNullException(nameof(connector)); + + public Task ExecuteAsync(JobExecutionContext context, CancellationToken cancellationToken) + => _connector.ParseAsync(context.Services, cancellationToken); +} + +internal sealed class CertInMapJob : IJob +{ + private readonly CertInConnector _connector; + + public CertInMapJob(CertInConnector connector) + => _connector = connector ?? throw new ArgumentNullException(nameof(connector)); + + public Task ExecuteAsync(JobExecutionContext context, CancellationToken cancellationToken) + => _connector.MapAsync(context.Services, cancellationToken); +} diff --git a/src/StellaOps.Feedser.Source.CertIn/StellaOps.Feedser.Source.CertIn.csproj b/src/StellaOps.Feedser.Source.CertIn/StellaOps.Feedser.Source.CertIn.csproj new file mode 100644 index 00000000..7e54853b --- /dev/null +++ b/src/StellaOps.Feedser.Source.CertIn/StellaOps.Feedser.Source.CertIn.csproj @@ -0,0 +1,16 @@ + + + + net10.0 + enable + enable + + + + + + + + + + diff --git a/src/StellaOps.Feedser.Source.CertIn/TASKS.md b/src/StellaOps.Feedser.Source.CertIn/TASKS.md new file mode 100644 index 00000000..c821b649 --- /dev/null +++ b/src/StellaOps.Feedser.Source.CertIn/TASKS.md @@ -0,0 +1,10 @@ +# TASKS +| Task | Owner(s) | Depends on | Notes | +|---|---|---|---| +|Index/detail crawler with windowing|BE-Conn-CertIn|Source.Common|**DONE** – index/detail fetch implemented with window overlap and pagination.| +|Extractor (title/CVEs/mitigation)|BE-Conn-CertIn|Source.Common|**DONE** – parser normalizes encodings, CVE lists, and mitigation text.| +|DTO validation and sanitizer|BE-Conn-CertIn, QA|Source.Common|**DONE** – HTML sanitizer produces DTO before persistence.| +|Canonical mapping (aliases, refs)|BE-Conn-CertIn|Models|**DONE** – mapper creates CERT-IN aliases plus typed references.| +|State/dedupe and fixtures|BE-Conn-CertIn, QA|Storage.Mongo|**DONE** – snapshot/resume tests cover dedupe and cursor handling.| +|Mark failure/backoff on fetch errors|BE-Conn-CertIn|Storage.Mongo|**DONE** – fetch pipeline marks failures/backoff with unit coverage.| +|Conditional fetch caching|BE-Conn-CertIn|Source.Common|**DONE** – connector reuses ETag/Last-Modified; tests verify not-modified flow.| diff --git a/src/StellaOps.Feedser.Source.Common.Tests/Common/CannedHttpMessageHandlerTests.cs b/src/StellaOps.Feedser.Source.Common.Tests/Common/CannedHttpMessageHandlerTests.cs new file mode 100644 index 00000000..ad0be9f5 --- /dev/null +++ b/src/StellaOps.Feedser.Source.Common.Tests/Common/CannedHttpMessageHandlerTests.cs @@ -0,0 +1,37 @@ +using System.Net; +using System.Net.Http; +using StellaOps.Feedser.Source.Common.Testing; + +namespace StellaOps.Feedser.Source.Common.Tests; + +public sealed class CannedHttpMessageHandlerTests +{ + [Fact] + public async Task SendAsync_RecordsRequestsAndSupportsFallback() + { + var handler = new CannedHttpMessageHandler(); + var requestUri = new Uri("https://example.test/api/resource"); + handler.AddResponse(HttpMethod.Get, requestUri, () => new HttpResponseMessage(HttpStatusCode.OK)); + handler.SetFallback(_ => new HttpResponseMessage(HttpStatusCode.NotFound)); + + using var client = handler.CreateClient(); + var firstResponse = await client.GetAsync(requestUri); + var secondResponse = await client.GetAsync(new Uri("https://example.test/other")); + + Assert.Equal(HttpStatusCode.OK, firstResponse.StatusCode); + Assert.Equal(HttpStatusCode.NotFound, secondResponse.StatusCode); + Assert.Equal(2, handler.Requests.Count); + handler.AssertNoPendingResponses(); + } + + [Fact] + public async Task AddException_ThrowsDuringSend() + { + var handler = new CannedHttpMessageHandler(); + var requestUri = new Uri("https://example.test/api/error"); + handler.AddException(HttpMethod.Get, requestUri, new InvalidOperationException("boom")); + + using var client = handler.CreateClient(); + await Assert.ThrowsAsync(() => client.GetAsync(requestUri)); + } +} diff --git a/src/StellaOps.Feedser.Source.Common.Tests/Common/HtmlContentSanitizerTests.cs b/src/StellaOps.Feedser.Source.Common.Tests/Common/HtmlContentSanitizerTests.cs new file mode 100644 index 00000000..4415c2c7 --- /dev/null +++ b/src/StellaOps.Feedser.Source.Common.Tests/Common/HtmlContentSanitizerTests.cs @@ -0,0 +1,31 @@ +using StellaOps.Feedser.Source.Common.Html; + +namespace StellaOps.Feedser.Source.Common.Tests; + +public sealed class HtmlContentSanitizerTests +{ + [Fact] + public void Sanitize_RemovesScriptAndDangerousAttributes() + { + var sanitizer = new HtmlContentSanitizer(); + var input = "
    link
    "; + + var sanitized = sanitizer.Sanitize(input, new Uri("https://example.test/base/")); + + Assert.DoesNotContain("script", sanitized, StringComparison.OrdinalIgnoreCase); + Assert.DoesNotContain("onclick", sanitized, StringComparison.OrdinalIgnoreCase); + Assert.Contains("https://example.test/foo", sanitized, StringComparison.Ordinal); + Assert.Contains("rel=\"noopener nofollow noreferrer\"", sanitized, StringComparison.Ordinal); + } + + [Fact] + public void Sanitize_PreservesBasicFormatting() + { + var sanitizer = new HtmlContentSanitizer(); + var input = "

    Hello world

    "; + + var sanitized = sanitizer.Sanitize(input); + + Assert.Equal("

    Hello world

    ", sanitized); + } +} diff --git a/src/StellaOps.Feedser.Source.Common.Tests/Common/PackageCoordinateHelperTests.cs b/src/StellaOps.Feedser.Source.Common.Tests/Common/PackageCoordinateHelperTests.cs new file mode 100644 index 00000000..3668af6e --- /dev/null +++ b/src/StellaOps.Feedser.Source.Common.Tests/Common/PackageCoordinateHelperTests.cs @@ -0,0 +1,41 @@ +using NuGet.Versioning; +using StellaOps.Feedser.Source.Common.Packages; + +namespace StellaOps.Feedser.Source.Common.Tests; + +public sealed class PackageCoordinateHelperTests +{ + [Fact] + public void TryParsePackageUrl_ReturnsCanonicalForm() + { + var success = PackageCoordinateHelper.TryParsePackageUrl("pkg:npm/@scope/example@1.0.0?env=prod", out var coordinates); + + Assert.True(success); + Assert.NotNull(coordinates); + Assert.Equal("pkg:npm/@scope/example@1.0.0?env=prod", coordinates!.Canonical); + Assert.Equal("npm", coordinates.Type); + Assert.Equal("example", coordinates.Name); + Assert.Equal("1.0.0", coordinates.Version); + Assert.Equal("prod", coordinates.Qualifiers["env"]); + } + + [Fact] + public void TryParseSemVer_NormalizesVersion() + { + var success = PackageCoordinateHelper.TryParseSemVer("1.2.3+build", out var version, out var normalized); + + Assert.True(success); + Assert.Equal(SemanticVersion.Parse("1.2.3"), version); + Assert.Equal("1.2.3", normalized); + } + + [Fact] + public void TryParseSemVerRange_SupportsCaret() + { + var success = PackageCoordinateHelper.TryParseSemVerRange("^1.2.3", out var range); + + Assert.True(success); + Assert.NotNull(range); + Assert.True(range!.Satisfies(NuGetVersion.Parse("1.3.0"))); + } +} diff --git a/src/StellaOps.Feedser.Source.Common.Tests/Common/PdfTextExtractorTests.cs b/src/StellaOps.Feedser.Source.Common.Tests/Common/PdfTextExtractorTests.cs new file mode 100644 index 00000000..b2da19a9 --- /dev/null +++ b/src/StellaOps.Feedser.Source.Common.Tests/Common/PdfTextExtractorTests.cs @@ -0,0 +1,21 @@ +using StellaOps.Feedser.Source.Common.Pdf; + +namespace StellaOps.Feedser.Source.Common.Tests; + +public sealed class PdfTextExtractorTests +{ + private const string SamplePdfBase64 = "JVBERi0xLjEKMSAwIG9iago8PCAvVHlwZSAvQ2F0YWxvZyAvUGFnZXMgMiAwIFIgPj4KZW5kb2JqCjIgMCBvYmoKPDwgL1R5cGUgL1BhZ2VzIC9LaWRzIFszIDAgUl0gL0NvdW50IDEgPj4KZW5kb2JqCjMgMCBvYmoKPDwgL1R5cGUgL1BhZ2UgL1BhcmVudCAyIDAgUiAvTWVkaWFCb3ggWzAgMCA2MTIgNzkyXSAvQ29udGVudHMgNCAwIFIgPj4KZW5kb2JqCjQgMCBvYmoKPDwgL0xlbmd0aCA0NCA+PgpzdHJlYW0KQlQKL0YxIDI0IFRmCjcyIDcyMCBUZAooSGVsbG8gV29ybGQpIFRqCkVUCmVuZHN0cmVhbQplbmRvYmoKNSAwIG9iago8PCAvVHlwZSAvRm9udCAvU3VidHlwZSAvVHlwZTEgL0Jhc2VGb250IC9IZWx2ZXRpY2EgPj4KZW5kb2JqCnhyZWYKMCA2CjAwMDAwMDAwMCA2NTUzNSBmIAowMDAwMDAwMTAgMDAwMDAgbiAKMDAwMDAwMDU2IDAwMDAwIG4gCjAwMDAwMDAxMTMgMDAwMDAgbiAKMDAwMDAwMDIxMCAwMDAwMCBuIAowMDAwMDAwMzExIDAwMDAwIG4gCnRyYWlsZXIKPDwgL1Jvb3QgMSAwIFIgL1NpemUgNiA+PgpzdGFydHhyZWYKMzc3CiUlRU9G"; + + [Fact] + public async Task ExtractTextAsync_ReturnsPageText() + { + var bytes = Convert.FromBase64String(SamplePdfBase64); + using var stream = new MemoryStream(bytes); + var extractor = new PdfTextExtractor(); + + var result = await extractor.ExtractTextAsync(stream, cancellationToken: CancellationToken.None); + + Assert.Contains("Hello World", result.Text); + Assert.Equal(1, result.PagesProcessed); + } +} diff --git a/src/StellaOps.Feedser.Source.Common.Tests/Common/SourceFetchServiceTests.cs b/src/StellaOps.Feedser.Source.Common.Tests/Common/SourceFetchServiceTests.cs new file mode 100644 index 00000000..ad2c3713 --- /dev/null +++ b/src/StellaOps.Feedser.Source.Common.Tests/Common/SourceFetchServiceTests.cs @@ -0,0 +1,36 @@ +using StellaOps.Feedser.Source.Common.Fetch; + +namespace StellaOps.Feedser.Source.Common.Tests; + +public sealed class SourceFetchServiceTests +{ + [Fact] + public void CreateHttpRequestMessage_DefaultsToJsonAccept() + { + var request = new SourceFetchRequest("client", "source", new Uri("https://example.test/data")); + + using var message = SourceFetchService.CreateHttpRequestMessage(request); + + Assert.Single(message.Headers.Accept); + Assert.Equal("application/json", message.Headers.Accept.First().MediaType); + } + + [Fact] + public void CreateHttpRequestMessage_UsesAcceptOverrides() + { + var request = new SourceFetchRequest("client", "source", new Uri("https://example.test/data")) + { + AcceptHeaders = new[] + { + "text/html", + "application/xhtml+xml;q=0.9", + } + }; + + using var message = SourceFetchService.CreateHttpRequestMessage(request); + + Assert.Equal(2, message.Headers.Accept.Count); + Assert.Contains(message.Headers.Accept, h => h.MediaType == "text/html"); + Assert.Contains(message.Headers.Accept, h => h.MediaType == "application/xhtml+xml"); + } +} diff --git a/src/StellaOps.Feedser.Source.Common.Tests/Common/TimeWindowCursorPlannerTests.cs b/src/StellaOps.Feedser.Source.Common.Tests/Common/TimeWindowCursorPlannerTests.cs new file mode 100644 index 00000000..9e6afb1e --- /dev/null +++ b/src/StellaOps.Feedser.Source.Common.Tests/Common/TimeWindowCursorPlannerTests.cs @@ -0,0 +1,87 @@ +using MongoDB.Bson; +using StellaOps.Feedser.Source.Common.Cursors; + +namespace StellaOps.Feedser.Source.Common.Tests; + +public sealed class TimeWindowCursorPlannerTests +{ + [Fact] + public void GetNextWindow_UsesInitialBackfillWhenStateEmpty() + { + var now = new DateTimeOffset(2024, 10, 1, 12, 0, 0, TimeSpan.Zero); + var options = new TimeWindowCursorOptions + { + WindowSize = TimeSpan.FromHours(4), + Overlap = TimeSpan.FromMinutes(15), + InitialBackfill = TimeSpan.FromDays(2), + MinimumWindowSize = TimeSpan.FromMinutes(1), + }; + + var window = TimeWindowCursorPlanner.GetNextWindow(now, null, options); + + Assert.Equal(now - options.InitialBackfill, window.Start); + Assert.Equal(window.Start + options.WindowSize, window.End); + } + + [Fact] + public void GetNextWindow_ClampsEndToNowWhenWindowExtendPastPresent() + { + var now = new DateTimeOffset(2024, 10, 10, 0, 0, 0, TimeSpan.Zero); + var options = new TimeWindowCursorOptions + { + WindowSize = TimeSpan.FromHours(6), + Overlap = TimeSpan.FromMinutes(30), + InitialBackfill = TimeSpan.FromDays(3), + MinimumWindowSize = TimeSpan.FromMinutes(1), + }; + + var previousEnd = now - TimeSpan.FromMinutes(10); + var state = new TimeWindowCursorState(previousEnd - options.WindowSize, previousEnd); + + var window = TimeWindowCursorPlanner.GetNextWindow(now, state, options); + + var expectedStart = previousEnd - options.Overlap; + var earliest = now - options.InitialBackfill; + if (expectedStart < earliest) + { + expectedStart = earliest; + } + + Assert.Equal(expectedStart, window.Start); + Assert.Equal(now, window.End); + } + + [Fact] + public void TimeWindowCursorState_RoundTripThroughBson() + { + var state = new TimeWindowCursorState( + new DateTimeOffset(2024, 9, 1, 0, 0, 0, TimeSpan.Zero), + new DateTimeOffset(2024, 9, 1, 6, 0, 0, TimeSpan.Zero)); + + var document = new BsonDocument + { + ["preserve"] = "value", + }; + + state.WriteTo(document); + var roundTripped = TimeWindowCursorState.FromBsonDocument(document); + + Assert.Equal(state.LastWindowStart, roundTripped.LastWindowStart); + Assert.Equal(state.LastWindowEnd, roundTripped.LastWindowEnd); + Assert.Equal("value", document["preserve"].AsString); + } + + [Fact] + public void PaginationPlanner_EnumeratesAdditionalPages() + { + var indices = PaginationPlanner.EnumerateAdditionalPages(4500, 2000).ToArray(); + Assert.Equal(new[] { 2000, 4000 }, indices); + } + + [Fact] + public void PaginationPlanner_ReturnsEmptyWhenSinglePage() + { + var indices = PaginationPlanner.EnumerateAdditionalPages(1000, 2000).ToArray(); + Assert.Empty(indices); + } +} diff --git a/src/StellaOps.Feedser.Source.Common.Tests/Common/UrlNormalizerTests.cs b/src/StellaOps.Feedser.Source.Common.Tests/Common/UrlNormalizerTests.cs new file mode 100644 index 00000000..3eeeef49 --- /dev/null +++ b/src/StellaOps.Feedser.Source.Common.Tests/Common/UrlNormalizerTests.cs @@ -0,0 +1,24 @@ +using StellaOps.Feedser.Source.Common.Url; + +namespace StellaOps.Feedser.Source.Common.Tests; + +public sealed class UrlNormalizerTests +{ + [Fact] + public void TryNormalize_ResolvesRelative() + { + var success = UrlNormalizer.TryNormalize("/foo/bar", new Uri("https://example.test/base/"), out var normalized); + + Assert.True(success); + Assert.Equal("https://example.test/foo/bar", normalized!.ToString()); + } + + [Fact] + public void TryNormalize_StripsFragment() + { + var success = UrlNormalizer.TryNormalize("https://example.test/path#section", null, out var normalized); + + Assert.True(success); + Assert.Equal("https://example.test/path", normalized!.ToString()); + } +} diff --git a/src/StellaOps.Feedser.Source.Common.Tests/Json/JsonSchemaValidatorTests.cs b/src/StellaOps.Feedser.Source.Common.Tests/Json/JsonSchemaValidatorTests.cs new file mode 100644 index 00000000..f676b87c --- /dev/null +++ b/src/StellaOps.Feedser.Source.Common.Tests/Json/JsonSchemaValidatorTests.cs @@ -0,0 +1,51 @@ +using System; +using System.Text.Json; +using Json.Schema; +using Microsoft.Extensions.Logging.Abstractions; +using StellaOps.Feedser.Source.Common.Json; + +namespace StellaOps.Feedser.Source.Common.Tests.Json; + +public sealed class JsonSchemaValidatorTests +{ + private static JsonSchema CreateSchema() + => JsonSchema.FromText(""" + { + "type": "object", + "properties": { + "id": { "type": "string" }, + "count": { "type": "integer", "minimum": 1 } + }, + "required": ["id", "count"], + "additionalProperties": false + } + """); + + [Fact] + public void Validate_AllowsDocumentsMatchingSchema() + { + var schema = CreateSchema(); + using var document = JsonDocument.Parse("""{"id":"abc","count":2}"""); + var validator = new JsonSchemaValidator(NullLogger.Instance); + + var exception = Record.Exception(() => validator.Validate(document, schema, "valid-doc")); + + Assert.Null(exception); + } + + [Fact] + public void Validate_ThrowsWithDetailedViolations() + { + var schema = CreateSchema(); + using var document = JsonDocument.Parse("""{"count":0,"extra":"nope"}"""); + var validator = new JsonSchemaValidator(NullLogger.Instance); + + var ex = Assert.Throws(() => validator.Validate(document, schema, "invalid-doc")); + + Assert.Equal("invalid-doc", ex.DocumentName); + Assert.NotEmpty(ex.Errors); + Assert.Contains(ex.Errors, error => error.Keyword == "required"); + Assert.Contains(ex.Errors, error => error.SchemaLocation.Contains("#/additionalProperties", StringComparison.Ordinal)); + Assert.Contains(ex.Errors, error => error.Keyword == "minimum"); + } +} diff --git a/src/StellaOps.Feedser.Source.Common.Tests/StellaOps.Feedser.Source.Common.Tests.csproj b/src/StellaOps.Feedser.Source.Common.Tests/StellaOps.Feedser.Source.Common.Tests.csproj new file mode 100644 index 00000000..43dcfcd7 --- /dev/null +++ b/src/StellaOps.Feedser.Source.Common.Tests/StellaOps.Feedser.Source.Common.Tests.csproj @@ -0,0 +1,10 @@ + + + net10.0 + enable + enable + + + + + diff --git a/src/StellaOps.Feedser.Source.Common.Tests/Xml/XmlSchemaValidatorTests.cs b/src/StellaOps.Feedser.Source.Common.Tests/Xml/XmlSchemaValidatorTests.cs new file mode 100644 index 00000000..5a2ab89f --- /dev/null +++ b/src/StellaOps.Feedser.Source.Common.Tests/Xml/XmlSchemaValidatorTests.cs @@ -0,0 +1,58 @@ +using System.IO; +using System.Xml; +using System.Xml.Linq; +using System.Xml.Schema; +using Microsoft.Extensions.Logging.Abstractions; +using FeedserXmlSchemaValidator = StellaOps.Feedser.Source.Common.Xml.XmlSchemaValidator; +using FeedserXmlSchemaValidationException = StellaOps.Feedser.Source.Common.Xml.XmlSchemaValidationException; + +namespace StellaOps.Feedser.Source.Common.Tests.Xml; + +public sealed class XmlSchemaValidatorTests +{ + private static XmlSchemaSet CreateSchema() + { + var set = new XmlSchemaSet(); + set.Add(string.Empty, XmlReader.Create(new StringReader(""" + + + + + + + + + + + """))); + set.CompilationSettings = new XmlSchemaCompilationSettings { EnableUpaCheck = true }; + set.Compile(); + return set; + } + + [Fact] + public void Validate_AllowsCompliantDocument() + { + var schemaSet = CreateSchema(); + var document = XDocument.Parse("abc3"); + var validator = new FeedserXmlSchemaValidator(NullLogger.Instance); + + var exception = Record.Exception(() => validator.Validate(document, schemaSet, "valid.xml")); + + Assert.Null(exception); + } + + [Fact] + public void Validate_ThrowsWithDetailedErrors() + { + var schemaSet = CreateSchema(); + var document = XDocument.Parse("missing-count"); + var validator = new FeedserXmlSchemaValidator(NullLogger.Instance); + + var ex = Assert.Throws(() => validator.Validate(document, schemaSet, "invalid.xml")); + + Assert.Equal("invalid.xml", ex.DocumentName); + Assert.NotEmpty(ex.Errors); + Assert.Contains(ex.Errors, error => error.Message.Contains("count", StringComparison.OrdinalIgnoreCase)); + } +} diff --git a/src/StellaOps.Feedser.Source.Common/AGENTS.md b/src/StellaOps.Feedser.Source.Common/AGENTS.md new file mode 100644 index 00000000..582d2bc2 --- /dev/null +++ b/src/StellaOps.Feedser.Source.Common/AGENTS.md @@ -0,0 +1,31 @@ +# AGENTS +## Role +Shared connector toolkit. Provides HTTP clients, retry/backoff, conditional GET (ETag/Last-Modified), schema validation, pagination helpers, clocks, and common DTO utilities for all connectors. +## Scope +- Typed HttpClient registrations with allowlisted hosts and timeouts. +- Request pipeline: retries with jitter, backoff on 429/5xx, rate-limit tracking per source. +- Conditional GET helpers (If-None-Match, If-Modified-Since), window cursors, and pagination iterators. +- Validators: JSON Schema, XML Schema (for example XmlSchemaValidator), and sanitizers. +- Content hashing and raw document capture helpers; metadata extraction (headers, status). +- HTML sanitization, URL normalization, and PDF-to-text extraction utilities for feeds that require cleanup before validation. +## Participants +- Source.* connectors (NVD, Red Hat, JVN, PSIRTs, CERTs, ICS). +- Storage.Mongo (document/dto repositories using shared shapes). +- Core (jobs schedule/trigger for connectors). +- QA (canned HTTP server harness, schema fixtures). +## Interfaces & contracts +- All network calls must pass through configured HttpClient with allowlist and sane timeouts; no direct new HttpClient(). +- Validators return detailed errors; invalid payloads quarantined and not mapped. +- Cursor helpers implement sliding windows and ID-based pagination; rely on IClock/TimeProvider for determinism. +- Strict provenance tags for extraction method: parser, oval, package.nevra, llm (gated). +## In/Out of scope +In: HTTP plumbing, validators, cursor/backoff utilities, hashing. +Out: connector-specific schemas/mapping rules, merge precedence. +## Observability & security expectations +- Metrics: SourceDiagnostics publishes `feedser.source.http.*` counters/histograms tagged with `feedser.source=` plus retries/failures; connector dashboards slice on that tag instead of bespoke metric names. +- Logs include uri, status, retries, etag; redact tokens and auth headers. +- Distributed tracing hooks and per-connector counters should be wired centrally for consistent observability. +## Tests +- Author and review coverage in `../StellaOps.Feedser.Source.Common.Tests`. +- Shared fixtures (e.g., `MongoIntegrationFixture`, `ConnectorTestHarness`) live in `../StellaOps.Feedser.Testing`. +- Keep fixtures deterministic; match new cases to real-world advisories or regression scenarios. diff --git a/src/StellaOps.Feedser.Source.Common/Cursors/PaginationPlanner.cs b/src/StellaOps.Feedser.Source.Common/Cursors/PaginationPlanner.cs new file mode 100644 index 00000000..b1eec949 --- /dev/null +++ b/src/StellaOps.Feedser.Source.Common/Cursors/PaginationPlanner.cs @@ -0,0 +1,29 @@ +namespace StellaOps.Feedser.Source.Common.Cursors; + +/// +/// Provides helpers for computing pagination start indices for sources that expose total result counts. +/// +public static class PaginationPlanner +{ + /// + /// Enumerates additional page start indices given the total result count returned by the source. + /// The first page (at ) is assumed to be already fetched. + /// + public static IEnumerable EnumerateAdditionalPages(int totalResults, int resultsPerPage, int firstPageStartIndex = 0) + { + if (totalResults <= 0 || resultsPerPage <= 0) + { + yield break; + } + + if (firstPageStartIndex < 0) + { + firstPageStartIndex = 0; + } + + for (var start = firstPageStartIndex + resultsPerPage; start < totalResults; start += resultsPerPage) + { + yield return start; + } + } +} diff --git a/src/StellaOps.Feedser.Source.Common/Cursors/TimeWindowCursorOptions.cs b/src/StellaOps.Feedser.Source.Common/Cursors/TimeWindowCursorOptions.cs new file mode 100644 index 00000000..a33c7216 --- /dev/null +++ b/src/StellaOps.Feedser.Source.Common/Cursors/TimeWindowCursorOptions.cs @@ -0,0 +1,43 @@ +namespace StellaOps.Feedser.Source.Common.Cursors; + +/// +/// Configuration applied when advancing sliding time-window cursors. +/// +public sealed class TimeWindowCursorOptions +{ + public TimeSpan WindowSize { get; init; } = TimeSpan.FromHours(4); + + public TimeSpan Overlap { get; init; } = TimeSpan.FromMinutes(5); + + public TimeSpan InitialBackfill { get; init; } = TimeSpan.FromDays(7); + + public TimeSpan MinimumWindowSize { get; init; } = TimeSpan.FromMinutes(1); + + public void EnsureValid() + { + if (WindowSize <= TimeSpan.Zero) + { + throw new InvalidOperationException("Window size must be positive."); + } + + if (Overlap < TimeSpan.Zero) + { + throw new InvalidOperationException("Window overlap cannot be negative."); + } + + if (Overlap >= WindowSize) + { + throw new InvalidOperationException("Window overlap must be less than the window size."); + } + + if (InitialBackfill <= TimeSpan.Zero) + { + throw new InvalidOperationException("Initial backfill must be positive."); + } + + if (MinimumWindowSize <= TimeSpan.Zero) + { + throw new InvalidOperationException("Minimum window size must be positive."); + } + } +} diff --git a/src/StellaOps.Feedser.Source.Common/Cursors/TimeWindowCursorPlanner.cs b/src/StellaOps.Feedser.Source.Common/Cursors/TimeWindowCursorPlanner.cs new file mode 100644 index 00000000..10b1b357 --- /dev/null +++ b/src/StellaOps.Feedser.Source.Common/Cursors/TimeWindowCursorPlanner.cs @@ -0,0 +1,50 @@ +namespace StellaOps.Feedser.Source.Common.Cursors; + +/// +/// Utility methods for computing sliding time-window ranges used by connectors. +/// +public static class TimeWindowCursorPlanner +{ + public static TimeWindow GetNextWindow(DateTimeOffset now, TimeWindowCursorState? state, TimeWindowCursorOptions options) + { + ArgumentNullException.ThrowIfNull(options); + options.EnsureValid(); + + var effectiveState = state ?? TimeWindowCursorState.Empty; + + var earliest = now - options.InitialBackfill; + var anchorEnd = effectiveState.LastWindowEnd ?? earliest; + if (anchorEnd < earliest) + { + anchorEnd = earliest; + } + + var start = anchorEnd - options.Overlap; + if (start < earliest) + { + start = earliest; + } + + var end = start + options.WindowSize; + if (end > now) + { + end = now; + } + + if (end <= start) + { + end = start + options.MinimumWindowSize; + if (end > now) + { + end = now; + } + } + + if (end <= start) + { + throw new InvalidOperationException("Unable to compute a non-empty time window with the provided options."); + } + + return new TimeWindow(start, end); + } +} diff --git a/src/StellaOps.Feedser.Source.Common/Cursors/TimeWindowCursorState.cs b/src/StellaOps.Feedser.Source.Common/Cursors/TimeWindowCursorState.cs new file mode 100644 index 00000000..5c835eb0 --- /dev/null +++ b/src/StellaOps.Feedser.Source.Common/Cursors/TimeWindowCursorState.cs @@ -0,0 +1,84 @@ +using MongoDB.Bson; + +namespace StellaOps.Feedser.Source.Common.Cursors; + +/// +/// Represents the persisted state of a sliding time-window cursor. +/// +public sealed record TimeWindowCursorState(DateTimeOffset? LastWindowStart, DateTimeOffset? LastWindowEnd) +{ + public static TimeWindowCursorState Empty { get; } = new(null, null); + + public TimeWindowCursorState WithWindow(TimeWindow window) + { + return new TimeWindowCursorState(window.Start, window.End); + } + + public BsonDocument ToBsonDocument(string startField = "windowStart", string endField = "windowEnd") + { + var document = new BsonDocument(); + WriteTo(document, startField, endField); + return document; + } + + public void WriteTo(BsonDocument document, string startField = "windowStart", string endField = "windowEnd") + { + ArgumentNullException.ThrowIfNull(document); + ArgumentException.ThrowIfNullOrEmpty(startField); + ArgumentException.ThrowIfNullOrEmpty(endField); + + document.Remove(startField); + document.Remove(endField); + + if (LastWindowStart.HasValue) + { + document[startField] = LastWindowStart.Value.UtcDateTime; + } + + if (LastWindowEnd.HasValue) + { + document[endField] = LastWindowEnd.Value.UtcDateTime; + } + } + + public static TimeWindowCursorState FromBsonDocument(BsonDocument? document, string startField = "windowStart", string endField = "windowEnd") + { + if (document is null) + { + return Empty; + } + + DateTimeOffset? start = null; + DateTimeOffset? end = null; + + if (document.TryGetValue(startField, out var startValue)) + { + start = ReadDateTimeOffset(startValue); + } + + if (document.TryGetValue(endField, out var endValue)) + { + end = ReadDateTimeOffset(endValue); + } + + return new TimeWindowCursorState(start, end); + } + + private static DateTimeOffset? ReadDateTimeOffset(BsonValue value) + { + return value.BsonType switch + { + BsonType.DateTime => DateTime.SpecifyKind(value.ToUniversalTime(), DateTimeKind.Utc), + BsonType.String when DateTimeOffset.TryParse(value.AsString, out var parsed) => parsed.ToUniversalTime(), + _ => null, + }; + } +} + +/// +/// Simple value object describing a time window. +/// +public readonly record struct TimeWindow(DateTimeOffset Start, DateTimeOffset End) +{ + public TimeSpan Duration => End - Start; +} diff --git a/src/StellaOps.Feedser.Source.Common/DocumentStatuses.cs b/src/StellaOps.Feedser.Source.Common/DocumentStatuses.cs new file mode 100644 index 00000000..6d265643 --- /dev/null +++ b/src/StellaOps.Feedser.Source.Common/DocumentStatuses.cs @@ -0,0 +1,27 @@ +namespace StellaOps.Feedser.Source.Common; + +/// +/// Well-known lifecycle statuses for raw source documents as they move through fetch/parse/map stages. +/// +public static class DocumentStatuses +{ + /// + /// Document captured from the upstream source and awaiting schema validation/parsing. + /// + public const string PendingParse = "pending-parse"; + + /// + /// Document parsed and sanitized; awaiting canonical mapping. + /// + public const string PendingMap = "pending-map"; + + /// + /// Document fully mapped to canonical advisories. + /// + public const string Mapped = "mapped"; + + /// + /// Document failed processing; requires manual intervention before retry. + /// + public const string Failed = "failed"; +} diff --git a/src/StellaOps.Feedser.Source.Common/Fetch/CryptoJitterSource.cs b/src/StellaOps.Feedser.Source.Common/Fetch/CryptoJitterSource.cs new file mode 100644 index 00000000..1c798550 --- /dev/null +++ b/src/StellaOps.Feedser.Source.Common/Fetch/CryptoJitterSource.cs @@ -0,0 +1,43 @@ +using System.Security.Cryptography; + +namespace StellaOps.Feedser.Source.Common.Fetch; + +/// +/// Jitter source backed by for thread-safe, high-entropy delays. +/// +public sealed class CryptoJitterSource : IJitterSource +{ + public TimeSpan Next(TimeSpan minInclusive, TimeSpan maxInclusive) + { + if (maxInclusive < minInclusive) + { + throw new ArgumentException("Max jitter must be greater than or equal to min jitter.", nameof(maxInclusive)); + } + + if (minInclusive < TimeSpan.Zero) + { + minInclusive = TimeSpan.Zero; + } + + if (maxInclusive == minInclusive) + { + return minInclusive; + } + + var minTicks = minInclusive.Ticks; + var maxTicks = maxInclusive.Ticks; + var range = maxTicks - minTicks; + + Span buffer = stackalloc byte[8]; + RandomNumberGenerator.Fill(buffer); + var sample = BitConverter.ToUInt64(buffer); + var ratio = sample / (double)ulong.MaxValue; + var jitterTicks = (long)Math.Round(range * ratio, MidpointRounding.AwayFromZero); + if (jitterTicks > range) + { + jitterTicks = range; + } + + return TimeSpan.FromTicks(minTicks + jitterTicks); + } +} diff --git a/src/StellaOps.Feedser.Source.Common/Fetch/IJitterSource.cs b/src/StellaOps.Feedser.Source.Common/Fetch/IJitterSource.cs new file mode 100644 index 00000000..6284e1b2 --- /dev/null +++ b/src/StellaOps.Feedser.Source.Common/Fetch/IJitterSource.cs @@ -0,0 +1,9 @@ +namespace StellaOps.Feedser.Source.Common.Fetch; + +/// +/// Produces random jitter durations used to decorrelate retries. +/// +public interface IJitterSource +{ + TimeSpan Next(TimeSpan minInclusive, TimeSpan maxInclusive); +} diff --git a/src/StellaOps.Feedser.Source.Common/Fetch/RawDocumentStorage.cs b/src/StellaOps.Feedser.Source.Common/Fetch/RawDocumentStorage.cs new file mode 100644 index 00000000..a87d4dc2 --- /dev/null +++ b/src/StellaOps.Feedser.Source.Common/Fetch/RawDocumentStorage.cs @@ -0,0 +1,90 @@ +using MongoDB.Bson; +using MongoDB.Driver; +using MongoDB.Driver.GridFS; + +namespace StellaOps.Feedser.Source.Common.Fetch; + +/// +/// Handles persistence of raw upstream documents in GridFS buckets for later parsing. +/// +public sealed class RawDocumentStorage +{ + private const string BucketName = "documents"; + + private readonly IMongoDatabase _database; + + public RawDocumentStorage(IMongoDatabase database) + { + _database = database ?? throw new ArgumentNullException(nameof(database)); + } + + private GridFSBucket CreateBucket() => new(_database, new GridFSBucketOptions + { + BucketName = BucketName, + WriteConcern = _database.Settings.WriteConcern, + ReadConcern = _database.Settings.ReadConcern, + }); + + public Task UploadAsync( + string sourceName, + string uri, + byte[] content, + string? contentType, + CancellationToken cancellationToken) + => UploadAsync(sourceName, uri, content, contentType, expiresAt: null, cancellationToken); + + public async Task UploadAsync( + string sourceName, + string uri, + byte[] content, + string? contentType, + DateTimeOffset? expiresAt, + CancellationToken cancellationToken) + { + ArgumentException.ThrowIfNullOrEmpty(sourceName); + ArgumentException.ThrowIfNullOrEmpty(uri); + ArgumentNullException.ThrowIfNull(content); + + var bucket = CreateBucket(); + var filename = $"{sourceName}/{Guid.NewGuid():N}"; + var metadata = new BsonDocument + { + ["sourceName"] = sourceName, + ["uri"] = uri, + }; + + if (!string.IsNullOrWhiteSpace(contentType)) + { + metadata["contentType"] = contentType; + } + + if (expiresAt.HasValue) + { + metadata["expiresAt"] = expiresAt.Value.UtcDateTime; + } + + return await bucket.UploadFromBytesAsync(filename, content, new GridFSUploadOptions + { + Metadata = metadata, + }, cancellationToken).ConfigureAwait(false); + } + + public Task DownloadAsync(ObjectId id, CancellationToken cancellationToken) + { + var bucket = CreateBucket(); + return bucket.DownloadAsBytesAsync(id, cancellationToken: cancellationToken); + } + + public async Task DeleteAsync(ObjectId id, CancellationToken cancellationToken) + { + var bucket = CreateBucket(); + try + { + await bucket.DeleteAsync(id, cancellationToken).ConfigureAwait(false); + } + catch (GridFSFileNotFoundException) + { + // Already removed; ignore. + } + } +} diff --git a/src/StellaOps.Feedser.Source.Common/Fetch/SourceFetchContentResult.cs b/src/StellaOps.Feedser.Source.Common/Fetch/SourceFetchContentResult.cs new file mode 100644 index 00000000..c8b63599 --- /dev/null +++ b/src/StellaOps.Feedser.Source.Common/Fetch/SourceFetchContentResult.cs @@ -0,0 +1,58 @@ +using System.Net; + +namespace StellaOps.Feedser.Source.Common.Fetch; + +/// +/// Result of fetching raw response content without persisting a document. +/// +public sealed record SourceFetchContentResult +{ + private SourceFetchContentResult( + HttpStatusCode statusCode, + byte[]? content, + bool notModified, + string? etag, + DateTimeOffset? lastModified, + string? contentType, + int attempts) + { + StatusCode = statusCode; + Content = content; + IsNotModified = notModified; + ETag = etag; + LastModified = lastModified; + ContentType = contentType; + Attempts = attempts; + } + + public HttpStatusCode StatusCode { get; } + + public byte[]? Content { get; } + + public bool IsSuccess => Content is not null; + + public bool IsNotModified { get; } + + public string? ETag { get; } + + public DateTimeOffset? LastModified { get; } + + public string? ContentType { get; } + + public int Attempts { get; } + + public static SourceFetchContentResult Success( + HttpStatusCode statusCode, + byte[] content, + string? etag, + DateTimeOffset? lastModified, + string? contentType, + int attempts) + => new(statusCode, content, notModified: false, etag, lastModified, contentType, attempts); + + public static SourceFetchContentResult NotModified(HttpStatusCode statusCode, int attempts) + => new(statusCode, null, notModified: true, etag: null, lastModified: null, contentType: null, attempts); + + public static SourceFetchContentResult Skipped(HttpStatusCode statusCode, int attempts) + => new(statusCode, null, notModified: false, etag: null, lastModified: null, contentType: null, attempts); +} diff --git a/src/StellaOps.Feedser.Source.Common/Fetch/SourceFetchRequest.cs b/src/StellaOps.Feedser.Source.Common/Fetch/SourceFetchRequest.cs new file mode 100644 index 00000000..1fed7047 --- /dev/null +++ b/src/StellaOps.Feedser.Source.Common/Fetch/SourceFetchRequest.cs @@ -0,0 +1,24 @@ +using System.Collections.Generic; +using System.Net.Http; + +namespace StellaOps.Feedser.Source.Common.Fetch; + +/// +/// Parameters describing a fetch operation for a source connector. +/// +public sealed record SourceFetchRequest( + string ClientName, + string SourceName, + HttpMethod Method, + Uri RequestUri, + IReadOnlyDictionary? Metadata = null, + string? ETag = null, + DateTimeOffset? LastModified = null, + TimeSpan? TimeoutOverride = null, + IReadOnlyList? AcceptHeaders = null) +{ + public SourceFetchRequest(string clientName, string sourceName, Uri requestUri) + : this(clientName, sourceName, HttpMethod.Get, requestUri) + { + } +} diff --git a/src/StellaOps.Feedser.Source.Common/Fetch/SourceFetchResult.cs b/src/StellaOps.Feedser.Source.Common/Fetch/SourceFetchResult.cs new file mode 100644 index 00000000..a1114097 --- /dev/null +++ b/src/StellaOps.Feedser.Source.Common/Fetch/SourceFetchResult.cs @@ -0,0 +1,34 @@ +using System.Net; +using StellaOps.Feedser.Storage.Mongo.Documents; + +namespace StellaOps.Feedser.Source.Common.Fetch; + +/// +/// Outcome of fetching a raw document from an upstream source. +/// +public sealed record SourceFetchResult +{ + private SourceFetchResult(HttpStatusCode statusCode, DocumentRecord? document, bool notModified) + { + StatusCode = statusCode; + Document = document; + IsNotModified = notModified; + } + + public HttpStatusCode StatusCode { get; } + + public DocumentRecord? Document { get; } + + public bool IsSuccess => Document is not null; + + public bool IsNotModified { get; } + + public static SourceFetchResult Success(DocumentRecord document, HttpStatusCode statusCode) + => new(statusCode, document, notModified: false); + + public static SourceFetchResult NotModified(HttpStatusCode statusCode) + => new(statusCode, null, notModified: true); + + public static SourceFetchResult Skipped(HttpStatusCode statusCode) + => new(statusCode, null, notModified: false); +} diff --git a/src/StellaOps.Feedser.Source.Common/Fetch/SourceFetchService.cs b/src/StellaOps.Feedser.Source.Common/Fetch/SourceFetchService.cs new file mode 100644 index 00000000..5322909a --- /dev/null +++ b/src/StellaOps.Feedser.Source.Common/Fetch/SourceFetchService.cs @@ -0,0 +1,313 @@ +using System.Diagnostics; +using System.Globalization; +using System.Linq; +using System.Net; +using System.Net.Http; +using System.Net.Http.Headers; +using System.Security.Cryptography; +using System.Text; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Options; +using MongoDB.Bson; +using StellaOps.Feedser.Source.Common.Http; +using StellaOps.Feedser.Source.Common.Telemetry; +using StellaOps.Feedser.Storage.Mongo; +using StellaOps.Feedser.Storage.Mongo.Documents; + +namespace StellaOps.Feedser.Source.Common.Fetch; + +/// +/// Executes HTTP fetches for connectors, capturing raw responses with metadata for downstream stages. +/// +public sealed class SourceFetchService +{ + private static readonly string[] DefaultAcceptHeaders = { "application/json" }; + + private readonly IHttpClientFactory _httpClientFactory; + private readonly RawDocumentStorage _rawDocumentStorage; + private readonly IDocumentStore _documentStore; + private readonly ILogger _logger; + private readonly TimeProvider _timeProvider; + private readonly IOptionsMonitor _httpClientOptions; + private readonly IOptions _storageOptions; + private readonly IJitterSource _jitterSource; + + public SourceFetchService( + IHttpClientFactory httpClientFactory, + RawDocumentStorage rawDocumentStorage, + IDocumentStore documentStore, + ILogger logger, + IJitterSource jitterSource, + TimeProvider? timeProvider = null, + IOptionsMonitor? httpClientOptions = null, + IOptions? storageOptions = null) + { + _httpClientFactory = httpClientFactory ?? throw new ArgumentNullException(nameof(httpClientFactory)); + _rawDocumentStorage = rawDocumentStorage ?? throw new ArgumentNullException(nameof(rawDocumentStorage)); + _documentStore = documentStore ?? throw new ArgumentNullException(nameof(documentStore)); + _logger = logger ?? throw new ArgumentNullException(nameof(logger)); + _jitterSource = jitterSource ?? throw new ArgumentNullException(nameof(jitterSource)); + _timeProvider = timeProvider ?? TimeProvider.System; + _httpClientOptions = httpClientOptions ?? throw new ArgumentNullException(nameof(httpClientOptions)); + _storageOptions = storageOptions ?? throw new ArgumentNullException(nameof(storageOptions)); + } + + public async Task FetchAsync(SourceFetchRequest request, CancellationToken cancellationToken) + { + ArgumentNullException.ThrowIfNull(request); + + using var activity = SourceDiagnostics.StartFetch(request.SourceName, request.RequestUri, request.Method.Method, request.ClientName); + var stopwatch = Stopwatch.StartNew(); + + try + { + var sendResult = await SendAsync(request, HttpCompletionOption.ResponseHeadersRead, cancellationToken).ConfigureAwait(false); + var response = sendResult.Response; + + using (response) + { + var duration = stopwatch.Elapsed; + activity?.SetTag("http.status_code", (int)response.StatusCode); + activity?.SetTag("http.retry.count", sendResult.Attempts - 1); + + var rateLimitRemaining = TryGetHeaderValue(response.Headers, "x-ratelimit-remaining"); + + if (response.StatusCode == HttpStatusCode.NotModified) + { + _logger.LogDebug("Source {Source} returned 304 Not Modified for {Uri}", request.SourceName, request.RequestUri); + SourceDiagnostics.RecordHttpRequest(request.SourceName, request.ClientName, response.StatusCode, sendResult.Attempts, duration, response.Content.Headers.ContentLength, rateLimitRemaining); + activity?.SetStatus(ActivityStatusCode.Ok); + return SourceFetchResult.NotModified(response.StatusCode); + } + + if (!response.IsSuccessStatusCode) + { + var body = await ReadResponsePreviewAsync(response, cancellationToken).ConfigureAwait(false); + SourceDiagnostics.RecordHttpRequest(request.SourceName, request.ClientName, response.StatusCode, sendResult.Attempts, duration, response.Content.Headers.ContentLength, rateLimitRemaining); + activity?.SetStatus(ActivityStatusCode.Error, body); + throw new HttpRequestException($"Fetch failed with status {(int)response.StatusCode} {response.StatusCode} from {request.RequestUri}. Body preview: {body}"); + } + + var contentBytes = await response.Content.ReadAsByteArrayAsync(cancellationToken).ConfigureAwait(false); + var sha256 = Convert.ToHexString(SHA256.HashData(contentBytes)).ToLowerInvariant(); + var fetchedAt = _timeProvider.GetUtcNow(); + var contentType = response.Content.Headers.ContentType?.ToString(); + var storageOptions = _storageOptions.Value; + var retention = storageOptions.RawDocumentRetention; + DateTimeOffset? expiresAt = null; + if (retention > TimeSpan.Zero) + { + var grace = storageOptions.RawDocumentRetentionTtlGrace >= TimeSpan.Zero + ? storageOptions.RawDocumentRetentionTtlGrace + : TimeSpan.Zero; + + try + { + expiresAt = fetchedAt.Add(retention).Add(grace); + } + catch (ArgumentOutOfRangeException) + { + expiresAt = DateTimeOffset.MaxValue; + } + } + + var gridFsId = await _rawDocumentStorage.UploadAsync( + request.SourceName, + request.RequestUri.ToString(), + contentBytes, + contentType, + expiresAt, + cancellationToken).ConfigureAwait(false); + + var headers = new Dictionary(StringComparer.OrdinalIgnoreCase); + foreach (var header in response.Headers) + { + headers[header.Key] = string.Join(",", header.Value); + } + + foreach (var header in response.Content.Headers) + { + headers[header.Key] = string.Join(",", header.Value); + } + + var metadata = request.Metadata is null + ? new Dictionary(StringComparer.Ordinal) + : new Dictionary(request.Metadata, StringComparer.Ordinal); + metadata["attempts"] = sendResult.Attempts.ToString(CultureInfo.InvariantCulture); + metadata["fetchedAt"] = fetchedAt.ToString("O"); + + var existing = await _documentStore.FindBySourceAndUriAsync(request.SourceName, request.RequestUri.ToString(), cancellationToken).ConfigureAwait(false); + var recordId = existing?.Id ?? Guid.NewGuid(); + + var record = new DocumentRecord( + recordId, + request.SourceName, + request.RequestUri.ToString(), + fetchedAt, + sha256, + DocumentStatuses.PendingParse, + contentType, + headers, + metadata, + response.Headers.ETag?.Tag, + response.Content.Headers.LastModified, + gridFsId, + expiresAt); + + var upserted = await _documentStore.UpsertAsync(record, cancellationToken).ConfigureAwait(false); + SourceDiagnostics.RecordHttpRequest(request.SourceName, request.ClientName, response.StatusCode, sendResult.Attempts, duration, contentBytes.LongLength, rateLimitRemaining); + activity?.SetStatus(ActivityStatusCode.Ok); + _logger.LogInformation("Fetched {Source} document {Uri} (sha256={Sha})", request.SourceName, request.RequestUri, sha256); + return SourceFetchResult.Success(upserted, response.StatusCode); + } + } + catch (Exception ex) when (ex is HttpRequestException or TaskCanceledException) + { + activity?.SetStatus(ActivityStatusCode.Error, ex.Message); + throw; + } + } + + public async Task FetchContentAsync(SourceFetchRequest request, CancellationToken cancellationToken) + { + ArgumentNullException.ThrowIfNull(request); + + using var activity = SourceDiagnostics.StartFetch(request.SourceName, request.RequestUri, request.Method.Method, request.ClientName); + var stopwatch = Stopwatch.StartNew(); + + try + { + _ = _httpClientOptions.Get(request.ClientName); + var sendResult = await SendAsync(request, HttpCompletionOption.ResponseHeadersRead, cancellationToken).ConfigureAwait(false); + var response = sendResult.Response; + + using (response) + { + var duration = stopwatch.Elapsed; + activity?.SetTag("http.status_code", (int)response.StatusCode); + activity?.SetTag("http.retry.count", sendResult.Attempts - 1); + + if (response.StatusCode == HttpStatusCode.NotModified) + { + _logger.LogDebug("Source {Source} returned 304 Not Modified for {Uri}", request.SourceName, request.RequestUri); + SourceDiagnostics.RecordHttpRequest(request.SourceName, request.ClientName, response.StatusCode, sendResult.Attempts, duration, response.Content.Headers.ContentLength, null); + activity?.SetStatus(ActivityStatusCode.Ok); + return SourceFetchContentResult.NotModified(response.StatusCode, sendResult.Attempts); + } + + if (!response.IsSuccessStatusCode) + { + var body = await ReadResponsePreviewAsync(response, cancellationToken).ConfigureAwait(false); + SourceDiagnostics.RecordHttpRequest(request.SourceName, request.ClientName, response.StatusCode, sendResult.Attempts, duration, response.Content.Headers.ContentLength, null); + activity?.SetStatus(ActivityStatusCode.Error, body); + throw new HttpRequestException($"Fetch failed with status {(int)response.StatusCode} {response.StatusCode} from {request.RequestUri}. Body preview: {body}"); + } + + var contentBytes = await response.Content.ReadAsByteArrayAsync(cancellationToken).ConfigureAwait(false); + SourceDiagnostics.RecordHttpRequest(request.SourceName, request.ClientName, response.StatusCode, sendResult.Attempts, duration, response.Content.Headers.ContentLength ?? contentBytes.LongLength, null); + activity?.SetStatus(ActivityStatusCode.Ok); + return SourceFetchContentResult.Success( + response.StatusCode, + contentBytes, + response.Headers.ETag?.Tag, + response.Content.Headers.LastModified, + response.Content.Headers.ContentType?.ToString(), + sendResult.Attempts); + } + } + catch (Exception ex) when (ex is HttpRequestException or TaskCanceledException) + { + activity?.SetStatus(ActivityStatusCode.Error, ex.Message); + throw; + } + } + + private async Task SendAsync(SourceFetchRequest request, HttpCompletionOption completionOption, CancellationToken cancellationToken) + { + var attemptCount = 0; + var options = _httpClientOptions.Get(request.ClientName); + + var response = await SourceRetryPolicy.SendWithRetryAsync( + () => CreateHttpRequestMessage(request), + async (httpRequest, ct) => + { + attemptCount++; + var client = _httpClientFactory.CreateClient(request.ClientName); + if (request.TimeoutOverride.HasValue) + { + client.Timeout = request.TimeoutOverride.Value; + } + + return await client.SendAsync(httpRequest, completionOption, ct).ConfigureAwait(false); + }, + maxAttempts: options.MaxAttempts, + baseDelay: options.BaseDelay, + _jitterSource, + context => SourceDiagnostics.RecordRetry( + request.SourceName, + request.ClientName, + context.Response?.StatusCode, + context.Attempt, + context.Delay), + cancellationToken).ConfigureAwait(false); + + return new SourceFetchSendResult(response, attemptCount); + } + + internal static HttpRequestMessage CreateHttpRequestMessage(SourceFetchRequest request) + { + var httpRequest = new HttpRequestMessage(request.Method, request.RequestUri); + var acceptValues = request.AcceptHeaders is { Count: > 0 } headers + ? headers + : DefaultAcceptHeaders; + + httpRequest.Headers.Accept.Clear(); + var added = false; + foreach (var mediaType in acceptValues) + { + if (string.IsNullOrWhiteSpace(mediaType)) + { + continue; + } + + if (MediaTypeWithQualityHeaderValue.TryParse(mediaType, out var headerValue)) + { + httpRequest.Headers.Accept.Add(headerValue); + added = true; + } + } + + if (!added) + { + httpRequest.Headers.Accept.Add(new MediaTypeWithQualityHeaderValue(DefaultAcceptHeaders[0])); + } + + return httpRequest; + } + + private static async Task ReadResponsePreviewAsync(HttpResponseMessage response, CancellationToken cancellationToken) + { + try + { + var buffer = await response.Content.ReadAsByteArrayAsync(cancellationToken).ConfigureAwait(false); + var preview = Encoding.UTF8.GetString(buffer); + return preview.Length > 256 ? preview[..256] : preview; + } + catch + { + return ""; + } + } + + private static string? TryGetHeaderValue(HttpResponseHeaders headers, string name) + { + if (headers.TryGetValues(name, out var values)) + { + return values.FirstOrDefault(); + } + + return null; + } + + private readonly record struct SourceFetchSendResult(HttpResponseMessage Response, int Attempts); +} diff --git a/src/StellaOps.Feedser.Source.Common/Fetch/SourceRetryPolicy.cs b/src/StellaOps.Feedser.Source.Common/Fetch/SourceRetryPolicy.cs new file mode 100644 index 00000000..4f634994 --- /dev/null +++ b/src/StellaOps.Feedser.Source.Common/Fetch/SourceRetryPolicy.cs @@ -0,0 +1,79 @@ +namespace StellaOps.Feedser.Source.Common.Fetch; + +/// +/// Provides retry/backoff behavior for source HTTP fetches. +/// +internal static class SourceRetryPolicy +{ + public static async Task SendWithRetryAsync( + Func requestFactory, + Func> sender, + int maxAttempts, + TimeSpan baseDelay, + IJitterSource jitterSource, + Action? onRetry, + CancellationToken cancellationToken) + { + ArgumentNullException.ThrowIfNull(requestFactory); + ArgumentNullException.ThrowIfNull(sender); + ArgumentNullException.ThrowIfNull(jitterSource); + + var attempt = 0; + + while (true) + { + attempt++; + using var request = requestFactory(); + HttpResponseMessage response; + + try + { + response = await sender(request, cancellationToken).ConfigureAwait(false); + } + catch (Exception ex) when (attempt < maxAttempts) + { + var delay = ComputeDelay(baseDelay, attempt, jitterSource: jitterSource); + onRetry?.Invoke(new SourceRetryAttemptContext(attempt, null, ex, delay)); + await Task.Delay(delay, cancellationToken).ConfigureAwait(false); + continue; + } + + if (NeedsRetry(response) && attempt < maxAttempts) + { + var delay = ComputeDelay(baseDelay, attempt, response.Headers.RetryAfter?.Delta, jitterSource); + onRetry?.Invoke(new SourceRetryAttemptContext(attempt, response, null, delay)); + response.Dispose(); + await Task.Delay(delay, cancellationToken).ConfigureAwait(false); + continue; + } + + return response; + } + } + + private static bool NeedsRetry(HttpResponseMessage response) + { + if (response.StatusCode == System.Net.HttpStatusCode.TooManyRequests) + { + return true; + } + + var status = (int)response.StatusCode; + return status >= 500 && status < 600; + } + + private static TimeSpan ComputeDelay(TimeSpan baseDelay, int attempt, TimeSpan? retryAfter = null, IJitterSource? jitterSource = null) + { + if (retryAfter.HasValue && retryAfter.Value > TimeSpan.Zero) + { + return retryAfter.Value; + } + + var exponential = TimeSpan.FromMilliseconds(baseDelay.TotalMilliseconds * Math.Pow(2, attempt - 1)); + var jitter = jitterSource?.Next(TimeSpan.FromMilliseconds(50), TimeSpan.FromMilliseconds(250)) + ?? TimeSpan.FromMilliseconds(Random.Shared.Next(50, 250)); + return exponential + jitter; + } +} + +internal readonly record struct SourceRetryAttemptContext(int Attempt, HttpResponseMessage? Response, Exception? Exception, TimeSpan Delay); diff --git a/src/StellaOps.Feedser.Source.Common/Html/HtmlContentSanitizer.cs b/src/StellaOps.Feedser.Source.Common/Html/HtmlContentSanitizer.cs new file mode 100644 index 00000000..61a6ecad --- /dev/null +++ b/src/StellaOps.Feedser.Source.Common/Html/HtmlContentSanitizer.cs @@ -0,0 +1,180 @@ +using System.Linq; +using AngleSharp.Dom; +using AngleSharp.Html.Parser; +using StellaOps.Feedser.Source.Common.Url; + +namespace StellaOps.Feedser.Source.Common.Html; + +/// +/// Sanitizes untrusted HTML fragments produced by upstream advisories. +/// Removes executable content, enforces an allowlist of elements, and normalizes anchor href values. +/// +public sealed class HtmlContentSanitizer +{ + private static readonly HashSet AllowedElements = new(StringComparer.OrdinalIgnoreCase) + { + "a", "abbr", "b", "body", "blockquote", "br", "code", "dd", "div", "dl", "dt", + "em", "html", "i", "li", "ol", "p", "pre", "s", "small", "span", + "strong", "sub", "sup", "table", "tbody", "td", "th", "thead", "tr", "ul" + }; + + private static readonly HashSet UrlAttributes = new(StringComparer.OrdinalIgnoreCase) + { + "href", "src", + }; + + private readonly HtmlParser _parser; + + public HtmlContentSanitizer() + { + _parser = new HtmlParser(new HtmlParserOptions + { + IsKeepingSourceReferences = false, + }); + } + + /// + /// Sanitizes and returns a safe fragment suitable for rendering. + /// + public string Sanitize(string? html, Uri? baseUri = null) + { + if (string.IsNullOrWhiteSpace(html)) + { + return string.Empty; + } + + var document = _parser.ParseDocument(html); + if (document.Body is null) + { + return string.Empty; + } + + foreach (var element in document.All.ToList()) + { + if (IsDangerous(element)) + { + element.Remove(); + continue; + } + + if (!AllowedElements.Contains(element.LocalName)) + { + var owner = element.Owner; + if (owner is null) + { + element.Remove(); + continue; + } + + var text = element.TextContent ?? string.Empty; + element.Replace(owner.CreateTextNode(text)); + continue; + } + + CleanAttributes(element, baseUri); + } + + var body = document.Body ?? document.DocumentElement; + if (body is null) + { + return string.Empty; + } + + var innerHtml = body.InnerHtml; + return string.IsNullOrWhiteSpace(innerHtml) ? string.Empty : innerHtml.Trim(); + } + + private static bool IsDangerous(IElement element) + { + if (string.Equals(element.LocalName, "script", StringComparison.OrdinalIgnoreCase) + || string.Equals(element.LocalName, "style", StringComparison.OrdinalIgnoreCase) + || string.Equals(element.LocalName, "iframe", StringComparison.OrdinalIgnoreCase) + || string.Equals(element.LocalName, "object", StringComparison.OrdinalIgnoreCase) + || string.Equals(element.LocalName, "embed", StringComparison.OrdinalIgnoreCase)) + { + return true; + } + + return false; + } + + private static void CleanAttributes(IElement element, Uri? baseUri) + { + if (element.Attributes is null || element.Attributes.Length == 0) + { + return; + } + + foreach (var attribute in element.Attributes.ToList()) + { + if (attribute.Name.StartsWith("on", StringComparison.OrdinalIgnoreCase)) + { + element.RemoveAttribute(attribute.Name); + continue; + } + + if (UrlAttributes.Contains(attribute.Name)) + { + NormalizeUrlAttribute(element, attribute, baseUri); + continue; + } + + if (!IsAttributeAllowed(element.LocalName, attribute.Name)) + { + element.RemoveAttribute(attribute.Name); + } + } + } + + private static bool IsAttributeAllowed(string elementName, string attributeName) + { + if (string.Equals(attributeName, "title", StringComparison.OrdinalIgnoreCase)) + { + return true; + } + + if (string.Equals(elementName, "a", StringComparison.OrdinalIgnoreCase) + && string.Equals(attributeName, "rel", StringComparison.OrdinalIgnoreCase)) + { + return true; + } + + if (string.Equals(elementName, "table", StringComparison.OrdinalIgnoreCase) + && (string.Equals(attributeName, "border", StringComparison.OrdinalIgnoreCase) + || string.Equals(attributeName, "cellpadding", StringComparison.OrdinalIgnoreCase) + || string.Equals(attributeName, "cellspacing", StringComparison.OrdinalIgnoreCase))) + { + return true; + } + + return false; + } + + private static void NormalizeUrlAttribute(IElement element, IAttr attribute, Uri? baseUri) + { + if (string.IsNullOrWhiteSpace(attribute.Value)) + { + element.RemoveAttribute(attribute.Name); + return; + } + + if (!UrlNormalizer.TryNormalize(attribute.Value, baseUri, out var normalized)) + { + element.RemoveAttribute(attribute.Name); + return; + } + + if (string.Equals(element.LocalName, "a", StringComparison.OrdinalIgnoreCase)) + { + element.SetAttribute("rel", "noopener nofollow noreferrer"); + } + + if (normalized is null) + { + element.RemoveAttribute(attribute.Name); + return; + } + + element.SetAttribute(attribute.Name, normalized.ToString()); + } +} diff --git a/src/StellaOps.Feedser.Source.Common/Http/AllowlistedHttpMessageHandler.cs b/src/StellaOps.Feedser.Source.Common/Http/AllowlistedHttpMessageHandler.cs new file mode 100644 index 00000000..327cab59 --- /dev/null +++ b/src/StellaOps.Feedser.Source.Common/Http/AllowlistedHttpMessageHandler.cs @@ -0,0 +1,36 @@ +using System.Net.Http.Headers; + +namespace StellaOps.Feedser.Source.Common.Http; + +/// +/// Delegating handler that enforces an allowlist of destination hosts for outbound requests. +/// +internal sealed class AllowlistedHttpMessageHandler : DelegatingHandler +{ + private readonly IReadOnlyCollection _allowedHosts; + + public AllowlistedHttpMessageHandler(SourceHttpClientOptions options) + { + ArgumentNullException.ThrowIfNull(options); + var snapshot = options.GetAllowedHostsSnapshot(); + if (snapshot.Count == 0) + { + throw new InvalidOperationException("Source HTTP client must configure at least one allowed host."); + } + + _allowedHosts = snapshot; + } + + protected override Task SendAsync(HttpRequestMessage request, CancellationToken cancellationToken) + { + ArgumentNullException.ThrowIfNull(request); + + var host = request.RequestUri?.Host; + if (string.IsNullOrWhiteSpace(host) || !_allowedHosts.Contains(host)) + { + throw new InvalidOperationException($"Request host '{host ?? ""}' is not allowlisted for this source."); + } + + return base.SendAsync(request, cancellationToken); + } +} diff --git a/src/StellaOps.Feedser.Source.Common/Http/ServiceCollectionExtensions.cs b/src/StellaOps.Feedser.Source.Common/Http/ServiceCollectionExtensions.cs new file mode 100644 index 00000000..a8eae3ff --- /dev/null +++ b/src/StellaOps.Feedser.Source.Common/Http/ServiceCollectionExtensions.cs @@ -0,0 +1,76 @@ +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Options; +using StellaOps.Feedser.Source.Common.Xml; + +namespace StellaOps.Feedser.Source.Common.Http; + +public static class ServiceCollectionExtensions +{ + /// + /// Registers a named HTTP client configured for a source connector with allowlisted hosts and sensible defaults. + /// + public static IHttpClientBuilder AddSourceHttpClient(this IServiceCollection services, string name, Action configure) + => services.AddSourceHttpClient(name, (_, options) => configure(options)); + + public static IHttpClientBuilder AddSourceHttpClient(this IServiceCollection services, string name, Action configure) + { + ArgumentNullException.ThrowIfNull(services); + ArgumentException.ThrowIfNullOrEmpty(name); + ArgumentNullException.ThrowIfNull(configure); + + services.AddOptions(name).Configure((options, sp) => configure(sp, options)); + + return services + .AddHttpClient(name) + .ConfigureHttpClient((sp, client) => + { + var options = sp.GetRequiredService>().Get(name); + + if (options.BaseAddress is not null) + { + client.BaseAddress = options.BaseAddress; + } + + client.Timeout = options.Timeout; + client.DefaultRequestHeaders.UserAgent.Clear(); + client.DefaultRequestHeaders.UserAgent.ParseAdd(options.UserAgent); + + foreach (var header in options.DefaultRequestHeaders) + { + client.DefaultRequestHeaders.TryAddWithoutValidation(header.Key, header.Value); + } + }) + .ConfigurePrimaryHttpMessageHandler((sp) => + { + var options = sp.GetRequiredService>().Get(name).Clone(); + return new HttpClientHandler + { + AllowAutoRedirect = options.AllowAutoRedirect, + AutomaticDecompression = System.Net.DecompressionMethods.All, + }; + }) + .AddHttpMessageHandler(sp => + { + var options = sp.GetRequiredService>().Get(name).Clone(); + return new AllowlistedHttpMessageHandler(options); + }); + } + + /// + /// Registers shared helpers used by source connectors. + /// + public static IServiceCollection AddSourceCommon(this IServiceCollection services) + { + ArgumentNullException.ThrowIfNull(services); + + services.AddSingleton(); + services.AddSingleton(sp => sp.GetRequiredService()); + services.AddSingleton(); + services.AddSingleton(sp => sp.GetRequiredService()); + services.AddSingleton(); + services.AddSingleton(); + services.AddSingleton(); + + return services; + } +} diff --git a/src/StellaOps.Feedser.Source.Common/Http/SourceHttpClientOptions.cs b/src/StellaOps.Feedser.Source.Common/Http/SourceHttpClientOptions.cs new file mode 100644 index 00000000..56cd30c6 --- /dev/null +++ b/src/StellaOps.Feedser.Source.Common/Http/SourceHttpClientOptions.cs @@ -0,0 +1,80 @@ +using System.Collections.ObjectModel; + +namespace StellaOps.Feedser.Source.Common.Http; + +/// +/// Configuration applied to named HTTP clients used by connectors. +/// +public sealed class SourceHttpClientOptions +{ + private readonly HashSet _allowedHosts = new(StringComparer.OrdinalIgnoreCase); + private readonly Dictionary _defaultHeaders = new(StringComparer.OrdinalIgnoreCase); + + /// + /// Gets or sets the base address used for relative requests. + /// + public Uri? BaseAddress { get; set; } + + /// + /// Gets or sets the client timeout. + /// + public TimeSpan Timeout { get; set; } = TimeSpan.FromSeconds(30); + + /// + /// Gets or sets the user-agent string applied to outgoing requests. + /// + public string UserAgent { get; set; } = "StellaOps.Feedser/1.0"; + + /// + /// Gets or sets whether redirects are allowed. Defaults to true. + /// + public bool AllowAutoRedirect { get; set; } = true; + + /// + /// Maximum number of retry attempts for transient failures. + /// + public int MaxAttempts { get; set; } = 3; + + /// + /// Base delay applied to the exponential backoff policy. + /// + public TimeSpan BaseDelay { get; set; } = TimeSpan.FromSeconds(2); + + /// + /// Hosts that this client is allowed to contact. + /// + public ISet AllowedHosts => _allowedHosts; + + /// + /// Default request headers appended to each outgoing request. + /// + public IDictionary DefaultRequestHeaders => _defaultHeaders; + + internal SourceHttpClientOptions Clone() + { + var clone = new SourceHttpClientOptions + { + BaseAddress = BaseAddress, + Timeout = Timeout, + UserAgent = UserAgent, + AllowAutoRedirect = AllowAutoRedirect, + MaxAttempts = MaxAttempts, + BaseDelay = BaseDelay, + }; + + foreach (var host in _allowedHosts) + { + clone.AllowedHosts.Add(host); + } + + foreach (var header in _defaultHeaders) + { + clone.DefaultRequestHeaders[header.Key] = header.Value; + } + + return clone; + } + + internal IReadOnlyCollection GetAllowedHostsSnapshot() + => new ReadOnlyCollection(_allowedHosts.ToArray()); +} diff --git a/src/StellaOps.Feedser.Source.Common/Json/IJsonSchemaValidator.cs b/src/StellaOps.Feedser.Source.Common/Json/IJsonSchemaValidator.cs new file mode 100644 index 00000000..47317d91 --- /dev/null +++ b/src/StellaOps.Feedser.Source.Common/Json/IJsonSchemaValidator.cs @@ -0,0 +1,9 @@ +using System.Text.Json; +using Json.Schema; + +namespace StellaOps.Feedser.Source.Common.Json; + +public interface IJsonSchemaValidator +{ + void Validate(JsonDocument document, JsonSchema schema, string documentName); +} diff --git a/src/StellaOps.Feedser.Source.Common/Json/JsonSchemaValidationError.cs b/src/StellaOps.Feedser.Source.Common/Json/JsonSchemaValidationError.cs new file mode 100644 index 00000000..f0de47f2 --- /dev/null +++ b/src/StellaOps.Feedser.Source.Common/Json/JsonSchemaValidationError.cs @@ -0,0 +1,7 @@ +namespace StellaOps.Feedser.Source.Common.Json; + +public sealed record JsonSchemaValidationError( + string InstanceLocation, + string SchemaLocation, + string Message, + string Keyword); diff --git a/src/StellaOps.Feedser.Source.Common/Json/JsonSchemaValidationException.cs b/src/StellaOps.Feedser.Source.Common/Json/JsonSchemaValidationException.cs new file mode 100644 index 00000000..c3ee6b2e --- /dev/null +++ b/src/StellaOps.Feedser.Source.Common/Json/JsonSchemaValidationException.cs @@ -0,0 +1,15 @@ +namespace StellaOps.Feedser.Source.Common.Json; + +public sealed class JsonSchemaValidationException : Exception +{ + public JsonSchemaValidationException(string documentName, IReadOnlyList errors) + : base($"JSON schema validation failed for '{documentName}'.") + { + DocumentName = documentName; + Errors = errors ?? Array.Empty(); + } + + public string DocumentName { get; } + + public IReadOnlyList Errors { get; } +} diff --git a/src/StellaOps.Feedser.Source.Common/Json/JsonSchemaValidator.cs b/src/StellaOps.Feedser.Source.Common/Json/JsonSchemaValidator.cs new file mode 100644 index 00000000..98408bf5 --- /dev/null +++ b/src/StellaOps.Feedser.Source.Common/Json/JsonSchemaValidator.cs @@ -0,0 +1,92 @@ +using System.Collections.Generic; +using System.Linq; +using System.Text.Json; +using Json.Schema; +using Microsoft.Extensions.Logging; + +namespace StellaOps.Feedser.Source.Common.Json; +public sealed class JsonSchemaValidator : IJsonSchemaValidator +{ + private readonly ILogger _logger; + private const int MaxLoggedErrors = 5; + + public JsonSchemaValidator(ILogger logger) + { + _logger = logger ?? throw new ArgumentNullException(nameof(logger)); + } + + public void Validate(JsonDocument document, JsonSchema schema, string documentName) + { + ArgumentNullException.ThrowIfNull(document); + ArgumentNullException.ThrowIfNull(schema); + ArgumentException.ThrowIfNullOrEmpty(documentName); + + var result = schema.Evaluate(document.RootElement, new EvaluationOptions + { + OutputFormat = OutputFormat.List, + RequireFormatValidation = true, + }); + + if (result.IsValid) + { + return; + } + + var errors = CollectErrors(result); + + if (errors.Count == 0) + { + _logger.LogWarning("Schema validation failed for {Document} with unknown errors", documentName); + throw new JsonSchemaValidationException(documentName, errors); + } + + foreach (var violation in errors.Take(MaxLoggedErrors)) + { + _logger.LogWarning( + "Schema violation for {Document} at {InstanceLocation} (keyword: {Keyword}): {Message}", + documentName, + string.IsNullOrEmpty(violation.InstanceLocation) ? "#" : violation.InstanceLocation, + violation.Keyword, + violation.Message); + } + + if (errors.Count > MaxLoggedErrors) + { + _logger.LogWarning("{Count} additional schema violations for {Document} suppressed", errors.Count - MaxLoggedErrors, documentName); + } + + throw new JsonSchemaValidationException(documentName, errors); + } + + private static IReadOnlyList CollectErrors(EvaluationResults result) + { + var errors = new List(); + Aggregate(result, errors); + return errors; + } + + private static void Aggregate(EvaluationResults node, List errors) + { + if (node.Errors is { Count: > 0 }) + { + foreach (var kvp in node.Errors) + { + errors.Add(new JsonSchemaValidationError( + node.InstanceLocation?.ToString() ?? string.Empty, + node.SchemaLocation?.ToString() ?? string.Empty, + kvp.Value, + kvp.Key)); + } + } + + if (node.Details is null) + { + return; + } + + foreach (var child in node.Details) + { + Aggregate(child, errors); + } + } +} diff --git a/src/StellaOps.Feedser.Source.Common/Packages/PackageCoordinateHelper.cs b/src/StellaOps.Feedser.Source.Common/Packages/PackageCoordinateHelper.cs new file mode 100644 index 00000000..7505f55d --- /dev/null +++ b/src/StellaOps.Feedser.Source.Common/Packages/PackageCoordinateHelper.cs @@ -0,0 +1,197 @@ +using System.Linq; +using System.Text; +using NuGet.Versioning; +using StellaOps.Feedser.Normalization.Identifiers; + +namespace StellaOps.Feedser.Source.Common.Packages; + +/// +/// Shared helpers for working with Package URLs and SemVer coordinates inside connectors. +/// +public static class PackageCoordinateHelper +{ + public static bool TryParsePackageUrl(string? value, out PackageCoordinates? coordinates) + { + coordinates = null; + if (!IdentifierNormalizer.TryNormalizePackageUrl(value, out var canonical, out var packageUrl) || packageUrl is null) + { + return false; + } + + var namespaceSegments = packageUrl.NamespaceSegments.ToArray(); + var subpathSegments = packageUrl.SubpathSegments.ToArray(); + var qualifiers = packageUrl.Qualifiers.ToDictionary(kvp => kvp.Key, kvp => kvp.Value, StringComparer.OrdinalIgnoreCase); + var canonicalRebuilt = BuildPackageUrl( + packageUrl.Type, + namespaceSegments, + packageUrl.Name, + packageUrl.Version, + qualifiers, + subpathSegments); + + coordinates = new PackageCoordinates( + Canonical: canonicalRebuilt, + Type: packageUrl.Type, + NamespaceSegments: namespaceSegments, + Name: packageUrl.Name, + Version: packageUrl.Version, + Qualifiers: qualifiers, + SubpathSegments: subpathSegments, + Original: packageUrl.Original); + return true; + } + + public static PackageCoordinates ParsePackageUrl(string value) + { + if (!TryParsePackageUrl(value, out var coordinates) || coordinates is null) + { + throw new FormatException($"Value '{value}' is not a valid Package URL"); + } + + return coordinates; + } + + public static bool TryParseSemVer(string? value, out SemanticVersion? version, out string? normalized) + { + version = null; + normalized = null; + + if (string.IsNullOrWhiteSpace(value)) + { + return false; + } + + if (!SemanticVersion.TryParse(value.Trim(), out var parsed)) + { + return false; + } + + version = parsed; + normalized = parsed.ToNormalizedString(); + return true; + } + + public static bool TryParseSemVerRange(string? value, out VersionRange? range) + { + range = null; + if (string.IsNullOrWhiteSpace(value)) + { + return false; + } + + var trimmed = value.Trim(); + if (trimmed.StartsWith("^", StringComparison.Ordinal)) + { + var baseSegment = trimmed[1..]; + if (!SemanticVersion.TryParse(baseSegment, out var baseVersion)) + { + return false; + } + + var upperBound = CalculateCaretUpperBound(baseVersion); + var caretExpression = $"[{baseVersion.ToNormalizedString()}, {upperBound.ToNormalizedString()})"; + if (VersionRange.TryParse(caretExpression, out var caretRange)) + { + range = caretRange; + return true; + } + + return false; + } + + if (!VersionRange.TryParse(trimmed, out var parsed)) + { + try + { + parsed = VersionRange.Parse(trimmed); + } + catch + { + return false; + } + } + + range = parsed; + return true; + } + + public static string BuildPackageUrl( + string type, + IReadOnlyList? namespaceSegments, + string name, + string? version = null, + IReadOnlyDictionary? qualifiers = null, + IReadOnlyList? subpathSegments = null) + { + ArgumentException.ThrowIfNullOrEmpty(type); + ArgumentException.ThrowIfNullOrEmpty(name); + + var builder = new StringBuilder("pkg:"); + builder.Append(type.Trim().ToLowerInvariant()); + builder.Append('/'); + + if (namespaceSegments is not null && namespaceSegments.Count > 0) + { + builder.Append(string.Join('/', namespaceSegments.Select(NormalizeSegment))); + builder.Append('/'); + } + + builder.Append(NormalizeSegment(name)); + + if (!string.IsNullOrWhiteSpace(version)) + { + builder.Append('@'); + builder.Append(version.Trim()); + } + + if (qualifiers is not null && qualifiers.Count > 0) + { + builder.Append('?'); + builder.Append(string.Join('&', qualifiers + .OrderBy(static kvp => kvp.Key, StringComparer.OrdinalIgnoreCase) + .Select(kvp => $"{NormalizeSegment(kvp.Key)}={NormalizeSegment(kvp.Value)}"))); + } + + if (subpathSegments is not null && subpathSegments.Count > 0) + { + builder.Append('#'); + builder.Append(string.Join('/', subpathSegments.Select(NormalizeSegment))); + } + + return builder.ToString(); + } + + private static string NormalizeSegment(string value) + { + ArgumentNullException.ThrowIfNull(value); + var trimmed = value.Trim(); + var unescaped = Uri.UnescapeDataString(trimmed); + var encoded = Uri.EscapeDataString(unescaped); + return encoded.Replace("%40", "@"); + } + + private static SemanticVersion CalculateCaretUpperBound(SemanticVersion baseVersion) + { + if (baseVersion.Major > 0) + { + return new SemanticVersion(baseVersion.Major + 1, 0, 0); + } + + if (baseVersion.Minor > 0) + { + return new SemanticVersion(0, baseVersion.Minor + 1, 0); + } + + return new SemanticVersion(0, 0, baseVersion.Patch + 1); + } +} + +public sealed record PackageCoordinates( + string Canonical, + string Type, + IReadOnlyList NamespaceSegments, + string Name, + string? Version, + IReadOnlyDictionary Qualifiers, + IReadOnlyList SubpathSegments, + string Original); diff --git a/src/StellaOps.Feedser.Source.Common/Pdf/PdfTextExtractor.cs b/src/StellaOps.Feedser.Source.Common/Pdf/PdfTextExtractor.cs new file mode 100644 index 00000000..a36d98de --- /dev/null +++ b/src/StellaOps.Feedser.Source.Common/Pdf/PdfTextExtractor.cs @@ -0,0 +1,184 @@ +using System; +using System.Collections.Generic; +using System.IO; +using System.Text.RegularExpressions; +using System.Text; +using UglyToad.PdfPig; +using UglyToad.PdfPig.Content; + +namespace StellaOps.Feedser.Source.Common.Pdf; + +/// +/// Extracts text from PDF advisories using UglyToad.PdfPig without requiring native dependencies. +/// +public sealed class PdfTextExtractor +{ + public async Task ExtractTextAsync(Stream pdfStream, PdfExtractionOptions? options = null, CancellationToken cancellationToken = default) + { + ArgumentNullException.ThrowIfNull(pdfStream); + options ??= PdfExtractionOptions.Default; + + using var buffer = new MemoryStream(); + await pdfStream.CopyToAsync(buffer, cancellationToken).ConfigureAwait(false); + var rawBytes = buffer.ToArray(); + buffer.Position = 0; + + using var document = PdfDocument.Open(buffer, new ParsingOptions + { + ClipPaths = true, + UseLenientParsing = true, + }); + + var builder = new StringBuilder(); + var pageCount = 0; + + var totalPages = document.NumberOfPages; + for (var index = 1; index <= totalPages; index++) + { + cancellationToken.ThrowIfCancellationRequested(); + + Page page; + try + { + page = document.GetPage(index); + } + catch (InvalidOperationException ex) when (ex.Message.Contains("empty stack", StringComparison.OrdinalIgnoreCase)) + { + continue; + } + + pageCount++; + if (options.MaxPages.HasValue && pageCount > options.MaxPages.Value) + { + break; + } + + if (pageCount > 1 && options.PageSeparator is not null) + { + builder.Append(options.PageSeparator); + } + + string text; + try + { + if (options.PreserveLayout) + { + text = page.Text; + } + else + { + text = FlattenWords(page.GetWords()); + } + } + catch (InvalidOperationException ex) when (ex.Message.Contains("empty stack", StringComparison.OrdinalIgnoreCase)) + { + try + { + text = FlattenWords(page.GetWords()); + } + catch + { + try + { + text = FlattenLetters(page.Letters); + } + catch + { + continue; + } + } + } + + if (!string.IsNullOrWhiteSpace(text)) + { + builder.AppendLine(text.Trim()); + } + } + + if (builder.Length == 0) + { + var raw = Encoding.ASCII.GetString(rawBytes); + var matches = Regex.Matches(raw, "\\(([^\\)]+)\\)", RegexOptions.CultureInvariant); + foreach (Match match in matches) + { + var value = match.Groups[1].Value; + if (!string.IsNullOrWhiteSpace(value)) + { + builder.AppendLine(value.Trim()); + } + } + + if (builder.Length > 0 && matches.Count > 0) + { + pageCount = Math.Max(pageCount, matches.Count); + } + } + else if (builder.Length > 0 && pageCount == 0) + { + pageCount = 1; + } + + return new PdfExtractionResult(builder.ToString().Trim(), pageCount); + } + + private static string FlattenWords(IEnumerable words) + { + var builder = new StringBuilder(); + var first = true; + foreach (var word in words) + { + if (string.IsNullOrWhiteSpace(word.Text)) + { + continue; + } + + if (!first) + { + builder.Append(' '); + } + + builder.Append(word.Text.Trim()); + first = false; + } + + return builder.ToString(); + } + + private static string FlattenLetters(IEnumerable letters) + { + var builder = new StringBuilder(); + foreach (var letter in letters) + { + if (letter.Value is null) + { + continue; + } + + builder.Append(letter.Value); + } + + return builder.ToString(); + } +} + +public sealed record PdfExtractionResult(string Text, int PagesProcessed); + +public sealed record PdfExtractionOptions +{ + public static PdfExtractionOptions Default { get; } = new(); + + /// + /// Maximum number of pages to read. Null reads the entire document. + /// + public int? MaxPages { get; init; } + + /// + /// When true, uses PdfPig's native layout text. When false, collapses to a single line per page. + /// + public bool PreserveLayout { get; init; } = true; + + /// + /// Separator inserted between pages. Null disables separators. + /// + public string? PageSeparator { get; init; } = "\n\n"; +} diff --git a/src/StellaOps.Feedser.Source.Common/Properties/AssemblyInfo.cs b/src/StellaOps.Feedser.Source.Common/Properties/AssemblyInfo.cs new file mode 100644 index 00000000..2d379827 --- /dev/null +++ b/src/StellaOps.Feedser.Source.Common/Properties/AssemblyInfo.cs @@ -0,0 +1,3 @@ +using System.Runtime.CompilerServices; + +[assembly: InternalsVisibleTo("StellaOps.Feedser.Source.Common.Tests")] diff --git a/src/StellaOps.Feedser.Source.Common/StellaOps.Feedser.Source.Common.csproj b/src/StellaOps.Feedser.Source.Common/StellaOps.Feedser.Source.Common.csproj new file mode 100644 index 00000000..5ec42c47 --- /dev/null +++ b/src/StellaOps.Feedser.Source.Common/StellaOps.Feedser.Source.Common.csproj @@ -0,0 +1,21 @@ + + + net10.0 + enable + enable + + + + + + + + + + + + + + + + diff --git a/src/StellaOps.Feedser.Source.Common/TASKS.md b/src/StellaOps.Feedser.Source.Common/TASKS.md new file mode 100644 index 00000000..c8f734e2 --- /dev/null +++ b/src/StellaOps.Feedser.Source.Common/TASKS.md @@ -0,0 +1,16 @@ +# TASKS +| Task | Owner(s) | Depends on | Notes | +|---|---|---|---| +|Register source HTTP clients with allowlists and timeouts|BE-Conn-Shared|Source.Common|**DONE** – `AddSourceHttpClient` wires named clients with host allowlists/timeouts.| +|Implement retry/backoff with jitter and 429 handling|BE-Conn-Shared|Source.Common|**DONE** – `SourceRetryPolicy` retries with 429/5xx handling and exponential backoff.| +|Conditional GET helpers (ETag/Last-Modified)|BE-Conn-Shared|Source.Common|**DONE** – `SourceFetchRequest` + fetch result propagate etag/last-modified for NotModified handling.| +|Windowed cursor and pagination utilities|BE-Conn-Shared|Source.Common|**DONE** – `TimeWindowCursorPlanner` + `PaginationPlanner` centralize sliding windows and additional page indices.| +|JSON/XML schema validators with rich errors|BE-Conn-Shared, QA|Source.Common|DONE – JsonSchemaValidator surfaces keyword/path/message details + tests.| +|Raw document capture helper|BE-Conn-Shared|Storage.Mongo|**DONE** – `SourceFetchService` stores raw payload + headers with sha256 metadata.| +|Canned HTTP test harness|QA|Source.Common|DONE – enriched `CannedHttpMessageHandler` with method-aware queues, request capture, fallbacks, and helpers + unit coverage.| +|HTML sanitization and URL normalization utilities|BE-Conn-Shared|Source.Common|DONE – `HtmlContentSanitizer` + `UrlNormalizer` provide safe fragments and canonical links for connectors.| +|PDF-to-text sandbox helper|BE-Conn-Shared|Source.Common|DONE – `PdfTextExtractor` uses PdfPig to yield deterministic text with options + tests.| +|PURL and SemVer helper library|BE-Conn-Shared|Models|DONE – `PackageCoordinateHelper` exposes normalized purl + SemVer parsing utilities backed by normalization.| +|Telemetry wiring (logs/metrics/traces)|BE-Conn-Shared|Observability|DONE – `SourceDiagnostics` emits Activity/Meter signals integrated into fetch pipeline and WebService OTEL setup.| +|Shared jitter source in retry policy|BE-Conn-Shared|Source.Common|**DONE** – `SourceRetryPolicy` now consumes injected `CryptoJitterSource` for thread-safe jitter.| +|Allow per-request Accept header overrides|BE-Conn-Shared|Source.Common|**DONE** – `SourceFetchRequest.AcceptHeaders` honored by `SourceFetchService` plus unit tests for overrides.| diff --git a/src/StellaOps.Feedser.Source.Common/Telemetry/SourceDiagnostics.cs b/src/StellaOps.Feedser.Source.Common/Telemetry/SourceDiagnostics.cs new file mode 100644 index 00000000..1f3a520c --- /dev/null +++ b/src/StellaOps.Feedser.Source.Common/Telemetry/SourceDiagnostics.cs @@ -0,0 +1,107 @@ +using System.Diagnostics; +using System.Diagnostics.Metrics; +using System.Net; + +namespace StellaOps.Feedser.Source.Common.Telemetry; + +/// +/// Central telemetry instrumentation for connector HTTP operations. +/// +public static class SourceDiagnostics +{ + public const string ActivitySourceName = "StellaOps.Feedser.Source"; + public const string MeterName = "StellaOps.Feedser.Source"; + + private static readonly ActivitySource ActivitySource = new(ActivitySourceName); + private static readonly Meter Meter = new(MeterName); + + private static readonly Counter HttpRequestCounter = Meter.CreateCounter("feedser.source.http.requests"); + private static readonly Counter HttpRetryCounter = Meter.CreateCounter("feedser.source.http.retries"); + private static readonly Counter HttpFailureCounter = Meter.CreateCounter("feedser.source.http.failures"); + private static readonly Counter HttpNotModifiedCounter = Meter.CreateCounter("feedser.source.http.not_modified"); + private static readonly Histogram HttpDuration = Meter.CreateHistogram("feedser.source.http.duration", unit: "ms"); + private static readonly Histogram HttpPayloadBytes = Meter.CreateHistogram("feedser.source.http.payload_bytes", unit: "byte"); + + public static Activity? StartFetch(string sourceName, Uri requestUri, string httpMethod, string? clientName) + { + var tags = new ActivityTagsCollection + { + { "feedser.source", sourceName }, + { "http.method", httpMethod }, + { "http.url", requestUri.ToString() }, + }; + + if (!string.IsNullOrWhiteSpace(clientName)) + { + tags.Add("http.client_name", clientName!); + } + + return ActivitySource.StartActivity("SourceFetch", ActivityKind.Client, parentContext: default, tags: tags); + } + + public static void RecordHttpRequest(string sourceName, string? clientName, HttpStatusCode statusCode, int attemptCount, TimeSpan duration, long? contentLength, string? rateLimitRemaining) + { + var tags = BuildDefaultTags(sourceName, clientName, statusCode, attemptCount); + HttpRequestCounter.Add(1, tags); + HttpDuration.Record(duration.TotalMilliseconds, tags); + + if (contentLength.HasValue && contentLength.Value >= 0) + { + HttpPayloadBytes.Record(contentLength.Value, tags); + } + + if (statusCode == HttpStatusCode.NotModified) + { + HttpNotModifiedCounter.Add(1, tags); + } + + if ((int)statusCode >= 500 || statusCode == HttpStatusCode.TooManyRequests) + { + HttpFailureCounter.Add(1, tags); + } + + if (!string.IsNullOrWhiteSpace(rateLimitRemaining) && long.TryParse(rateLimitRemaining, out var remaining)) + { + tags.Add("http.rate_limit.remaining", remaining); + } + } + + public static void RecordRetry(string sourceName, string? clientName, HttpStatusCode? statusCode, int attempt, TimeSpan delay) + { + var tags = new TagList + { + { "feedser.source", sourceName }, + { "http.retry_attempt", attempt }, + { "http.retry_delay_ms", delay.TotalMilliseconds }, + }; + + if (clientName is not null) + { + tags.Add("http.client_name", clientName); + } + + if (statusCode.HasValue) + { + tags.Add("http.status_code", (int)statusCode.Value); + } + + HttpRetryCounter.Add(1, tags); + } + + private static TagList BuildDefaultTags(string sourceName, string? clientName, HttpStatusCode statusCode, int attemptCount) + { + var tags = new TagList + { + { "feedser.source", sourceName }, + { "http.status_code", (int)statusCode }, + { "http.attempts", attemptCount }, + }; + + if (clientName is not null) + { + tags.Add("http.client_name", clientName); + } + + return tags; + } +} diff --git a/src/StellaOps.Feedser.Source.Common/Testing/CannedHttpMessageHandler.cs b/src/StellaOps.Feedser.Source.Common/Testing/CannedHttpMessageHandler.cs new file mode 100644 index 00000000..76c65f7e --- /dev/null +++ b/src/StellaOps.Feedser.Source.Common/Testing/CannedHttpMessageHandler.cs @@ -0,0 +1,210 @@ +using System.Collections.Concurrent; +using System.Net; +using System.Net.Http; +using System.Text; + +namespace StellaOps.Feedser.Source.Common.Testing; + +/// +/// Deterministic HTTP handler used by tests to supply canned responses keyed by request URI and method. +/// Tracks requests for assertions and supports fallbacks/exceptions. +/// +public sealed class CannedHttpMessageHandler : HttpMessageHandler +{ + private readonly ConcurrentDictionary>> _responses = + new(RequestKeyComparer.Instance); + + private readonly ConcurrentQueue _requests = new(); + + private Func? _fallback; + + /// + /// Recorded requests in arrival order. + /// + public IReadOnlyCollection Requests => _requests.ToArray(); + + /// + /// Registers a canned response for a GET request to . + /// + public void AddResponse(Uri requestUri, Func factory) + => AddResponse(HttpMethod.Get, requestUri, _ => factory()); + + /// + /// Registers a canned response for the specified method and URI. + /// + public void AddResponse(HttpMethod method, Uri requestUri, Func factory) + => AddResponse(method, requestUri, _ => factory()); + + /// + /// Registers a canned response using the full request context. + /// + public void AddResponse(HttpMethod method, Uri requestUri, Func factory) + { + ArgumentNullException.ThrowIfNull(method); + ArgumentNullException.ThrowIfNull(requestUri); + ArgumentNullException.ThrowIfNull(factory); + + var key = new RequestKey(method, requestUri); + var queue = _responses.GetOrAdd(key, static _ => new ConcurrentQueue>()); + queue.Enqueue(factory); + } + + /// + /// Registers an exception to be thrown for the specified request. + /// + public void AddException(HttpMethod method, Uri requestUri, Exception exception) + { + ArgumentNullException.ThrowIfNull(exception); + AddResponse(method, requestUri, _ => throw exception); + } + + /// + /// Registers a fallback used when no specific response is queued for a request. + /// + public void SetFallback(Func fallback) + { + ArgumentNullException.ThrowIfNull(fallback); + _fallback = fallback; + } + + /// + /// Clears registered responses and captured requests. + /// + public void Clear() + { + _responses.Clear(); + while (_requests.TryDequeue(out _)) + { + } + _fallback = null; + } + + /// + /// Throws if any responses remain queued. + /// + public void AssertNoPendingResponses() + { + foreach (var queue in _responses.Values) + { + if (!queue.IsEmpty) + { + throw new InvalidOperationException("Not all canned responses were consumed."); + } + } + } + + /// + /// Creates an wired to this handler. + /// + public HttpClient CreateClient() + => new(this, disposeHandler: false) + { + Timeout = TimeSpan.FromSeconds(10), + }; + + protected override Task SendAsync(HttpRequestMessage request, CancellationToken cancellationToken) + { + if (request.RequestUri is null) + { + throw new InvalidOperationException("Request URI is required for canned responses."); + } + + var key = new RequestKey(request.Method ?? HttpMethod.Get, request.RequestUri); + var factory = DequeueFactory(key); + + if (factory is null) + { + if (_fallback is null) + { + throw new InvalidOperationException($"No canned response registered for {request.Method} {request.RequestUri}."); + } + + factory = _fallback; + } + + var snapshot = CaptureRequest(request); + _requests.Enqueue(snapshot); + + var response = factory(request); + response.RequestMessage ??= request; + return Task.FromResult(response); + } + + private Func? DequeueFactory(RequestKey key) + { + if (_responses.TryGetValue(key, out var queue) && queue.TryDequeue(out var factory)) + { + return factory; + } + + return null; + } + + private static CannedRequestRecord CaptureRequest(HttpRequestMessage request) + { + var headers = new Dictionary(StringComparer.OrdinalIgnoreCase); + foreach (var header in request.Headers) + { + headers[header.Key] = string.Join(',', header.Value); + } + + if (request.Content is not null) + { + foreach (var header in request.Content.Headers) + { + headers[header.Key] = string.Join(',', header.Value); + } + } + + return new CannedRequestRecord( + Timestamp: DateTimeOffset.UtcNow, + Method: request.Method ?? HttpMethod.Get, + Uri: request.RequestUri!, + Headers: headers); + } + + private readonly record struct RequestKey(HttpMethod Method, string Uri) + { + public RequestKey(HttpMethod method, Uri uri) + : this(method, uri.ToString()) + { + } + + public bool Equals(RequestKey other) + => string.Equals(Method.Method, other.Method.Method, StringComparison.OrdinalIgnoreCase) + && string.Equals(Uri, other.Uri, StringComparison.OrdinalIgnoreCase); + + public override int GetHashCode() + { + var methodHash = StringComparer.OrdinalIgnoreCase.GetHashCode(Method.Method); + var uriHash = StringComparer.OrdinalIgnoreCase.GetHashCode(Uri); + return HashCode.Combine(methodHash, uriHash); + } + } + + private sealed class RequestKeyComparer : IEqualityComparer + { + public static readonly RequestKeyComparer Instance = new(); + + public bool Equals(RequestKey x, RequestKey y) => x.Equals(y); + + public int GetHashCode(RequestKey obj) => obj.GetHashCode(); + } + + public readonly record struct CannedRequestRecord(DateTimeOffset Timestamp, HttpMethod Method, Uri Uri, IReadOnlyDictionary Headers); + + private static HttpResponseMessage BuildTextResponse(HttpStatusCode statusCode, string content, string contentType) + { + var message = new HttpResponseMessage(statusCode) + { + Content = new StringContent(content, Encoding.UTF8, contentType), + }; + return message; + } + + public void AddJsonResponse(Uri requestUri, string json, HttpStatusCode statusCode = HttpStatusCode.OK) + => AddResponse(requestUri, () => BuildTextResponse(statusCode, json, "application/json")); + + public void AddTextResponse(Uri requestUri, string content, string contentType = "text/plain", HttpStatusCode statusCode = HttpStatusCode.OK) + => AddResponse(requestUri, () => BuildTextResponse(statusCode, content, contentType)); +} diff --git a/src/StellaOps.Feedser.Source.Common/Url/UrlNormalizer.cs b/src/StellaOps.Feedser.Source.Common/Url/UrlNormalizer.cs new file mode 100644 index 00000000..a9c85cf2 --- /dev/null +++ b/src/StellaOps.Feedser.Source.Common/Url/UrlNormalizer.cs @@ -0,0 +1,62 @@ +namespace StellaOps.Feedser.Source.Common.Url; + +/// +/// Utilities for normalizing URLs from upstream feeds. +/// +public static class UrlNormalizer +{ + /// + /// Attempts to normalize relative to . + /// Removes fragments and enforces HTTPS when possible. + /// + public static bool TryNormalize(string? value, Uri? baseUri, out Uri? normalized, bool stripFragment = true, bool forceHttps = false) + { + normalized = null; + if (string.IsNullOrWhiteSpace(value)) + { + return false; + } + + if (!Uri.TryCreate(value.Trim(), UriKind.RelativeOrAbsolute, out var candidate)) + { + return false; + } + + if (!candidate.IsAbsoluteUri) + { + if (baseUri is null) + { + return false; + } + + if (!Uri.TryCreate(baseUri, candidate, out candidate)) + { + return false; + } + } + + if (forceHttps && string.Equals(candidate.Scheme, Uri.UriSchemeHttp, StringComparison.OrdinalIgnoreCase)) + { + candidate = new UriBuilder(candidate) { Scheme = Uri.UriSchemeHttps, Port = candidate.IsDefaultPort ? -1 : candidate.Port }.Uri; + } + + if (stripFragment && !string.IsNullOrEmpty(candidate.Fragment)) + { + var builder = new UriBuilder(candidate) { Fragment = string.Empty }; + candidate = builder.Uri; + } + + normalized = candidate; + return true; + } + + public static Uri NormalizeOrThrow(string value, Uri? baseUri = null, bool stripFragment = true, bool forceHttps = false) + { + if (!TryNormalize(value, baseUri, out var normalized, stripFragment, forceHttps) || normalized is null) + { + throw new FormatException($"Value '{value}' is not a valid URI"); + } + + return normalized; + } +} diff --git a/src/StellaOps.Feedser.Source.Common/Xml/IXmlSchemaValidator.cs b/src/StellaOps.Feedser.Source.Common/Xml/IXmlSchemaValidator.cs new file mode 100644 index 00000000..cb340779 --- /dev/null +++ b/src/StellaOps.Feedser.Source.Common/Xml/IXmlSchemaValidator.cs @@ -0,0 +1,9 @@ +using System.Xml.Linq; +using System.Xml.Schema; + +namespace StellaOps.Feedser.Source.Common.Xml; + +public interface IXmlSchemaValidator +{ + void Validate(XDocument document, XmlSchemaSet schemaSet, string documentName); +} diff --git a/src/StellaOps.Feedser.Source.Common/Xml/XmlSchemaValidationError.cs b/src/StellaOps.Feedser.Source.Common/Xml/XmlSchemaValidationError.cs new file mode 100644 index 00000000..34ee3b2e --- /dev/null +++ b/src/StellaOps.Feedser.Source.Common/Xml/XmlSchemaValidationError.cs @@ -0,0 +1,3 @@ +namespace StellaOps.Feedser.Source.Common.Xml; + +public sealed record XmlSchemaValidationError(string Message, string? Location); diff --git a/src/StellaOps.Feedser.Source.Common/Xml/XmlSchemaValidationException.cs b/src/StellaOps.Feedser.Source.Common/Xml/XmlSchemaValidationException.cs new file mode 100644 index 00000000..a8b8cb3a --- /dev/null +++ b/src/StellaOps.Feedser.Source.Common/Xml/XmlSchemaValidationException.cs @@ -0,0 +1,18 @@ +using System; +using System.Collections.Generic; + +namespace StellaOps.Feedser.Source.Common.Xml; + +public sealed class XmlSchemaValidationException : Exception +{ + public XmlSchemaValidationException(string documentName, IReadOnlyList errors) + : base($"XML schema validation failed for '{documentName}'.") + { + DocumentName = documentName; + Errors = errors ?? Array.Empty(); + } + + public string DocumentName { get; } + + public IReadOnlyList Errors { get; } +} diff --git a/src/StellaOps.Feedser.Source.Common/Xml/XmlSchemaValidator.cs b/src/StellaOps.Feedser.Source.Common/Xml/XmlSchemaValidator.cs new file mode 100644 index 00000000..5ea71951 --- /dev/null +++ b/src/StellaOps.Feedser.Source.Common/Xml/XmlSchemaValidator.cs @@ -0,0 +1,71 @@ +using System; +using System.Collections.Generic; +using System.Xml.Linq; +using System.Xml.Schema; +using Microsoft.Extensions.Logging; + +namespace StellaOps.Feedser.Source.Common.Xml; + +public sealed class XmlSchemaValidator : IXmlSchemaValidator +{ + private readonly ILogger _logger; + + public XmlSchemaValidator(ILogger logger) + { + _logger = logger ?? throw new ArgumentNullException(nameof(logger)); + } + + public void Validate(XDocument document, XmlSchemaSet schemaSet, string documentName) + { + ArgumentNullException.ThrowIfNull(document); + ArgumentNullException.ThrowIfNull(schemaSet); + ArgumentException.ThrowIfNullOrWhiteSpace(documentName); + + var errors = new List(); + + void Handler(object? sender, ValidationEventArgs args) + { + if (args is null) + { + return; + } + + var location = FormatLocation(args.Exception); + errors.Add(new XmlSchemaValidationError(args.Message, location)); + } + + try + { + document.Validate(schemaSet, Handler, addSchemaInfo: true); + } + catch (System.Xml.Schema.XmlSchemaValidationException ex) + { + var location = FormatLocation(ex); + errors.Add(new XmlSchemaValidationError(ex.Message, location)); + } + + if (errors.Count > 0) + { + var exception = new XmlSchemaValidationException(documentName, errors); + _logger.LogError(exception, "XML schema validation failed for {DocumentName}", documentName); + throw exception; + } + + _logger.LogDebug("XML schema validation succeeded for {DocumentName}", documentName); + } + + private static string? FormatLocation(System.Xml.Schema.XmlSchemaException? exception) + { + if (exception is null) + { + return null; + } + + if (exception.LineNumber <= 0) + { + return null; + } + + return $"line {exception.LineNumber}, position {exception.LinePosition}"; + } +} diff --git a/src/StellaOps.Feedser.Source.Cve.Tests/Cve/CveConnectorTests.cs b/src/StellaOps.Feedser.Source.Cve.Tests/Cve/CveConnectorTests.cs new file mode 100644 index 00000000..3193153c --- /dev/null +++ b/src/StellaOps.Feedser.Source.Cve.Tests/Cve/CveConnectorTests.cs @@ -0,0 +1,130 @@ +using System.Net; +using System.Net.Http; +using System.Text; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Logging.Abstractions; +using Microsoft.Extensions.Options; +using MongoDB.Bson; +using MongoDB.Driver; +using StellaOps.Feedser.Models; +using StellaOps.Feedser.Source.Common.Fetch; +using StellaOps.Feedser.Source.Common.Testing; +using StellaOps.Feedser.Source.Cve.Configuration; +using StellaOps.Feedser.Testing; +using StellaOps.Feedser.Storage.Mongo; +using StellaOps.Feedser.Storage.Mongo.Advisories; +using StellaOps.Feedser.Storage.Mongo.Documents; +using StellaOps.Feedser.Storage.Mongo.Dtos; + +namespace StellaOps.Feedser.Source.Cve.Tests; + +[Collection("mongo-fixture")] +public sealed class CveConnectorTests : IAsyncLifetime +{ + private readonly MongoIntegrationFixture _fixture; + private ConnectorTestHarness? _harness; + + public CveConnectorTests(MongoIntegrationFixture fixture) + { + _fixture = fixture; + } + + [Fact] + public async Task FetchParseMap_EmitsCanonicalAdvisory() + { + var initialTime = new DateTimeOffset(2024, 10, 1, 0, 0, 0, TimeSpan.Zero); + await EnsureHarnessAsync(initialTime); + var harness = _harness!; + + var since = initialTime - TimeSpan.FromDays(30); + var listUri = new Uri($"https://cve.test/api/cve?time_modified.gte={Uri.EscapeDataString(since.ToString("O"))}&time_modified.lte={Uri.EscapeDataString(initialTime.ToString("O"))}&page=1&size=5"); + harness.Handler.AddJsonResponse(listUri, ReadFixture("Fixtures/cve-list.json")); + harness.Handler.SetFallback(request => + { + if (request.RequestUri is null) + { + return new HttpResponseMessage(HttpStatusCode.NotFound); + } + + if (request.RequestUri.AbsoluteUri.Equals("https://cve.test/api/cve/CVE-2024-0001", StringComparison.OrdinalIgnoreCase)) + { + return new HttpResponseMessage(HttpStatusCode.OK) + { + Content = new StringContent(ReadFixture("Fixtures/cve-CVE-2024-0001.json"), Encoding.UTF8, "application/json") + }; + } + + return new HttpResponseMessage(HttpStatusCode.NotFound); + }); + + var connector = new CveConnectorPlugin().Create(harness.ServiceProvider); + + await connector.FetchAsync(harness.ServiceProvider, CancellationToken.None); + await connector.ParseAsync(harness.ServiceProvider, CancellationToken.None); + await connector.MapAsync(harness.ServiceProvider, CancellationToken.None); + + var advisoryStore = harness.ServiceProvider.GetRequiredService(); + var advisory = await advisoryStore.FindAsync("CVE-2024-0001", CancellationToken.None); + Assert.NotNull(advisory); + + var snapshot = SnapshotSerializer.ToSnapshot(advisory!).Replace("\r\n", "\n").TrimEnd(); + var expected = ReadFixture("Fixtures/expected-CVE-2024-0001.json").Replace("\r\n", "\n").TrimEnd(); + + if (!string.Equals(expected, snapshot, StringComparison.Ordinal)) + { + var actualPath = Path.Combine(AppContext.BaseDirectory, "Fixtures", "expected-CVE-2024-0001.actual.json"); + Directory.CreateDirectory(Path.GetDirectoryName(actualPath)!); + File.WriteAllText(actualPath, snapshot); + } + + Assert.Equal(expected, snapshot); + harness.Handler.AssertNoPendingResponses(); + } + + private async Task EnsureHarnessAsync(DateTimeOffset initialTime) + { + if (_harness is not null) + { + return; + } + + var harness = new ConnectorTestHarness(_fixture, initialTime, CveOptions.HttpClientName); + await harness.EnsureServiceProviderAsync(services => + { + services.AddLogging(builder => builder.AddProvider(NullLoggerProvider.Instance)); + services.AddCveConnector(options => + { + options.BaseEndpoint = new Uri("https://cve.test/api/", UriKind.Absolute); + options.ApiOrg = "test-org"; + options.ApiUser = "test-user"; + options.ApiKey = "test-key"; + options.InitialBackfill = TimeSpan.FromDays(30); + options.PageSize = 5; + options.MaxPagesPerFetch = 2; + options.RequestDelay = TimeSpan.Zero; + }); + }); + + _harness = harness; + } + + private static string ReadFixture(string relativePath) + { + var path = Path.Combine(AppContext.BaseDirectory, relativePath); + return File.ReadAllText(path); + } + + public async Task InitializeAsync() + { + await Task.CompletedTask; + } + + public async Task DisposeAsync() + { + if (_harness is not null) + { + await _harness.DisposeAsync(); + } + } +} diff --git a/src/StellaOps.Feedser.Source.Cve.Tests/Fixtures/cve-CVE-2024-0001.json b/src/StellaOps.Feedser.Source.Cve.Tests/Fixtures/cve-CVE-2024-0001.json new file mode 100644 index 00000000..b9b89bfc --- /dev/null +++ b/src/StellaOps.Feedser.Source.Cve.Tests/Fixtures/cve-CVE-2024-0001.json @@ -0,0 +1,72 @@ +{ + "dataType": "CVE_RECORD", + "dataVersion": "5.0", + "cveMetadata": { + "cveId": "CVE-2024-0001", + "assignerShortName": "ExampleOrg", + "state": "PUBLISHED", + "dateReserved": "2024-01-01T00:00:00Z", + "datePublished": "2024-09-10T12:00:00Z", + "dateUpdated": "2024-09-15T12:00:00Z" + }, + "containers": { + "cna": { + "title": "Example Product Remote Code Execution", + "descriptions": [ + { + "lang": "en", + "value": "An example vulnerability allowing remote attackers to execute arbitrary code." + } + ], + "affected": [ + { + "vendor": "ExampleVendor", + "product": "ExampleProduct", + "platform": "linux", + "defaultStatus": "affected", + "versions": [ + { + "status": "affected", + "version": "1.0.0", + "lessThan": "1.2.0", + "versionType": "semver" + }, + { + "status": "unaffected", + "version": "1.2.0", + "versionType": "semver" + } + ] + } + ], + "references": [ + { + "url": "https://example.com/security/advisory", + "name": "Vendor Advisory", + "tags": [ + "vendor-advisory" + ] + }, + { + "url": "https://cve.example.com/CVE-2024-0001", + "tags": [ + "third-party-advisory" + ] + } + ], + "metrics": [ + { + "cvssV3_1": { + "version": "3.1", + "vectorString": "CVSS:3.1/AV:N/AC:L/PR:N/UI:N/S:U/C:H/I:H/A:H", + "baseScore": 9.8, + "baseSeverity": "CRITICAL" + } + } + ], + "aliases": [ + "GHSA-xxxx-yyyy-zzzz" + ] + } + } +} diff --git a/src/StellaOps.Feedser.Source.Cve.Tests/Fixtures/cve-list.json b/src/StellaOps.Feedser.Source.Cve.Tests/Fixtures/cve-list.json new file mode 100644 index 00000000..c2c15927 --- /dev/null +++ b/src/StellaOps.Feedser.Source.Cve.Tests/Fixtures/cve-list.json @@ -0,0 +1,18 @@ +{ + "dataType": "CVE_RECORD_LIST", + "dataVersion": "5.0", + "data": [ + { + "cveMetadata": { + "cveId": "CVE-2024-0001", + "state": "PUBLISHED", + "dateUpdated": "2024-09-15T12:00:00Z" + } + } + ], + "pagination": { + "page": 1, + "totalCount": 1, + "itemsPerPage": 5 + } +} diff --git a/src/StellaOps.Feedser.Source.Cve.Tests/Fixtures/expected-CVE-2024-0001.json b/src/StellaOps.Feedser.Source.Cve.Tests/Fixtures/expected-CVE-2024-0001.json new file mode 100644 index 00000000..37ad3acb --- /dev/null +++ b/src/StellaOps.Feedser.Source.Cve.Tests/Fixtures/expected-CVE-2024-0001.json @@ -0,0 +1,163 @@ +{ + "advisoryKey": "CVE-2024-0001", + "affectedPackages": [ + { + "identifier": "examplevendor:exampleproduct", + "platform": "linux", + "provenance": [ + { + "fieldMask": [], + "kind": "affected", + "recordedAt": "2024-10-01T00:00:00+00:00", + "source": "cve", + "value": "examplevendor:exampleproduct" + } + ], + "statuses": [ + { + "provenance": { + "fieldMask": [], + "kind": "affected-status", + "recordedAt": "2024-10-01T00:00:00+00:00", + "source": "cve", + "value": "examplevendor:exampleproduct" + }, + "status": "affected" + }, + { + "provenance": { + "fieldMask": [], + "kind": "affected-status", + "recordedAt": "2024-10-01T00:00:00+00:00", + "source": "cve", + "value": "examplevendor:exampleproduct" + }, + "status": "not_affected" + } + ], + "type": "vendor", + "versionRanges": [ + { + "fixedVersion": "1.2.0", + "introducedVersion": "1.0.0", + "lastAffectedVersion": null, + "primitives": { + "evr": null, + "hasVendorExtensions": true, + "nevra": null, + "semVer": null, + "vendorExtensions": { + "vendor": "ExampleVendor", + "product": "ExampleProduct", + "platform": "linux" + } + }, + "provenance": { + "fieldMask": [], + "kind": "affected-range", + "recordedAt": "2024-10-01T00:00:00+00:00", + "source": "cve", + "value": "examplevendor:exampleproduct" + }, + "rangeExpression": "version=1.0.0, < 1.2.0", + "rangeKind": "semver" + }, + { + "fixedVersion": null, + "introducedVersion": "1.2.0", + "lastAffectedVersion": null, + "primitives": { + "evr": null, + "hasVendorExtensions": true, + "nevra": null, + "semVer": null, + "vendorExtensions": { + "vendor": "ExampleVendor", + "product": "ExampleProduct", + "platform": "linux" + } + }, + "provenance": { + "fieldMask": [], + "kind": "affected-range", + "recordedAt": "2024-10-01T00:00:00+00:00", + "source": "cve", + "value": "examplevendor:exampleproduct" + }, + "rangeExpression": "version=1.2.0", + "rangeKind": "semver" + } + ] + } + ], + "aliases": [ + "CVE-2024-0001", + "GHSA-xxxx-yyyy-zzzz" + ], + "cvssMetrics": [ + { + "baseScore": 9.8, + "baseSeverity": "critical", + "provenance": { + "fieldMask": [], + "kind": "cvss", + "recordedAt": "2024-10-01T00:00:00+00:00", + "source": "cve", + "value": "cve/CVE-2024-0001" + }, + "vector": "CVSS:3.1/AV:N/AC:L/PR:N/UI:N/S:U/C:H/I:H/A:H", + "version": "3.1" + } + ], + "exploitKnown": false, + "language": "en", + "modified": "2024-09-15T12:00:00+00:00", + "provenance": [ + { + "fieldMask": [], + "kind": "document", + "recordedAt": "2024-10-01T00:00:00+00:00", + "source": "cve", + "value": "cve/CVE-2024-0001" + }, + { + "fieldMask": [], + "kind": "mapping", + "recordedAt": "2024-10-01T00:00:00+00:00", + "source": "cve", + "value": "CVE-2024-0001" + } + ], + "published": "2024-09-10T12:00:00+00:00", + "references": [ + { + "kind": "third-party-advisory", + "provenance": { + "fieldMask": [], + "kind": "reference", + "recordedAt": "2024-10-01T00:00:00+00:00", + "source": "cve", + "value": "https://cve.example.com/CVE-2024-0001" + }, + "sourceTag": null, + "summary": null, + "url": "https://cve.example.com/CVE-2024-0001" + }, + { + "kind": "vendor-advisory", + "provenance": { + "fieldMask": [], + "kind": "reference", + "recordedAt": "2024-10-01T00:00:00+00:00", + "source": "cve", + "value": "https://example.com/security/advisory" + }, + "sourceTag": "Vendor Advisory", + "summary": null, + "url": "https://example.com/security/advisory" + } + ], + "severity": "critical", + "summary": "An example vulnerability allowing remote attackers to execute arbitrary code.", + "title": "Example Product Remote Code Execution" +} diff --git a/src/StellaOps.Feedser.Source.Cve.Tests/StellaOps.Feedser.Source.Cve.Tests.csproj b/src/StellaOps.Feedser.Source.Cve.Tests/StellaOps.Feedser.Source.Cve.Tests.csproj new file mode 100644 index 00000000..c6a19365 --- /dev/null +++ b/src/StellaOps.Feedser.Source.Cve.Tests/StellaOps.Feedser.Source.Cve.Tests.csproj @@ -0,0 +1,17 @@ + + + net10.0 + enable + enable + + + + + + + + + + + + diff --git a/src/StellaOps.Feedser.Source.Cve/AGENTS.md b/src/StellaOps.Feedser.Source.Cve/AGENTS.md new file mode 100644 index 00000000..2d7c464a --- /dev/null +++ b/src/StellaOps.Feedser.Source.Cve/AGENTS.md @@ -0,0 +1,38 @@ +# AGENTS +## Role +Create a dedicated CVE connector when we need raw CVE stream ingestion outside of NVD/OSV/National feeds (e.g., CVE JSON 5 API or CNA disclosures). + +## Scope +- Determine whether this connector should consume the official CVE JSON 5 API, CNA disclosures, or another stream. +- Implement fetch/windowing aligned with CVE publication cadence; manage cursors for incremental backfills. +- Parse CVE payloads into DTOs capturing descriptions, affected vendors/products, references, and metrics. +- Map CVEs into canonical `Advisory` records (aliases, references, affected packages, range primitives). +- Deliver deterministic fixtures/tests for fetch/parse/map lifecycle. + +## Participants +- `Source.Common` (HTTP/fetch utilities, DTO storage). +- `Storage.Mongo` (raw/document/DTO/advisory stores & source state). +- `Feedser.Models` (canonical data model). +- `Feedser.Testing` (integration fixtures, snapshot helpers). + +## Interfaces & Contracts +- Job kinds: `cve:fetch`, `cve:parse`, `cve:map`. +- Persist upstream metadata (e.g., `If-Modified-Since`, `cveMetadataDate`) for incremental fetching. +- Aliases must include primary CVE ID along with CNA-specific identifiers when available. + +## In/Out of scope +In scope: +- Core pipeline for CVE ingestion with provenance/range primitives. + +Out of scope: +- Downstream impact scoring or enrichment (handled by other teams). + +## Observability & Security Expectations +- Log fetch batch sizes, update timestamps, and mapping counts. +- Handle rate limits politely with exponential backoff. +- Sanitize and validate payloads before persistence. + +## Tests +- Add `StellaOps.Feedser.Source.Cve.Tests` with canned CVE JSON fixtures covering fetch/parse/map. +- Snapshot canonical advisories; include env flag for fixture regeneration. +- Ensure deterministic ordering and timestamp handling. diff --git a/src/StellaOps.Feedser.Source.Cve/Configuration/CveOptions.cs b/src/StellaOps.Feedser.Source.Cve/Configuration/CveOptions.cs new file mode 100644 index 00000000..8d335291 --- /dev/null +++ b/src/StellaOps.Feedser.Source.Cve/Configuration/CveOptions.cs @@ -0,0 +1,100 @@ +using System; +using System.Diagnostics.CodeAnalysis; + +namespace StellaOps.Feedser.Source.Cve.Configuration; + +public sealed class CveOptions +{ + public static string HttpClientName => "source.cve"; + + public Uri BaseEndpoint { get; set; } = new("https://cveawg.mitre.org/api/", UriKind.Absolute); + + /// + /// CVE Services requires an organisation identifier for authenticated requests. + /// + public string ApiOrg { get; set; } = string.Empty; + + /// + /// CVE Services user identifier. Typically the username registered with the CVE Program. + /// + public string ApiUser { get; set; } = string.Empty; + + /// + /// API key issued by the CVE Program for the configured organisation/user pair. + /// + public string ApiKey { get; set; } = string.Empty; + + /// + /// Results fetched per page when querying CVE Services. Valid range 1-500. + /// + public int PageSize { get; set; } = 200; + + /// + /// Maximum number of pages to fetch in a single run. Guards against runaway backfills. + /// + public int MaxPagesPerFetch { get; set; } = 5; + + /// + /// Sliding look-back window when no previous cursor is available. + /// + public TimeSpan InitialBackfill { get; set; } = TimeSpan.FromDays(30); + + /// + /// Delay between paginated requests to respect API throttling guidance. + /// + public TimeSpan RequestDelay { get; set; } = TimeSpan.FromMilliseconds(250); + + /// + /// Backoff applied when the connector encounters an unrecoverable failure. + /// + public TimeSpan FailureBackoff { get; set; } = TimeSpan.FromMinutes(10); + + [MemberNotNull(nameof(BaseEndpoint), nameof(ApiOrg), nameof(ApiUser), nameof(ApiKey))] + public void Validate() + { + if (BaseEndpoint is null || !BaseEndpoint.IsAbsoluteUri) + { + throw new InvalidOperationException("BaseEndpoint must be an absolute URI."); + } + + if (string.IsNullOrWhiteSpace(ApiOrg)) + { + throw new InvalidOperationException("ApiOrg must be provided."); + } + + if (string.IsNullOrWhiteSpace(ApiUser)) + { + throw new InvalidOperationException("ApiUser must be provided."); + } + + if (string.IsNullOrWhiteSpace(ApiKey)) + { + throw new InvalidOperationException("ApiKey must be provided."); + } + + if (PageSize is < 1 or > 500) + { + throw new InvalidOperationException("PageSize must be between 1 and 500."); + } + + if (MaxPagesPerFetch <= 0) + { + throw new InvalidOperationException("MaxPagesPerFetch must be a positive integer."); + } + + if (InitialBackfill < TimeSpan.Zero) + { + throw new InvalidOperationException("InitialBackfill cannot be negative."); + } + + if (RequestDelay < TimeSpan.Zero) + { + throw new InvalidOperationException("RequestDelay cannot be negative."); + } + + if (FailureBackoff <= TimeSpan.Zero) + { + throw new InvalidOperationException("FailureBackoff must be greater than zero."); + } + } +} diff --git a/src/StellaOps.Feedser.Source.Cve/CveConnector.cs b/src/StellaOps.Feedser.Source.Cve/CveConnector.cs new file mode 100644 index 00000000..9eaa4ba7 --- /dev/null +++ b/src/StellaOps.Feedser.Source.Cve/CveConnector.cs @@ -0,0 +1,398 @@ +using System.Collections.Generic; +using System.Globalization; +using System.Linq; +using System.Net.Http; +using System.Text.Json; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Options; +using MongoDB.Bson; +using StellaOps.Feedser.Models; +using StellaOps.Feedser.Normalization.Text; +using StellaOps.Feedser.Source.Common; +using StellaOps.Feedser.Source.Common.Fetch; +using StellaOps.Feedser.Source.Cve.Configuration; +using StellaOps.Feedser.Source.Cve.Internal; +using StellaOps.Feedser.Storage.Mongo; +using StellaOps.Feedser.Storage.Mongo.Advisories; +using StellaOps.Feedser.Storage.Mongo.Documents; +using StellaOps.Feedser.Storage.Mongo.Dtos; +using StellaOps.Plugin; + +namespace StellaOps.Feedser.Source.Cve; + +public sealed class CveConnector : IFeedConnector +{ + private static readonly JsonSerializerOptions SerializerOptions = new(JsonSerializerDefaults.Web) + { + PropertyNameCaseInsensitive = true, + WriteIndented = false, + }; + + private readonly SourceFetchService _fetchService; + private readonly RawDocumentStorage _rawDocumentStorage; + private readonly IDocumentStore _documentStore; + private readonly IDtoStore _dtoStore; + private readonly IAdvisoryStore _advisoryStore; + private readonly ISourceStateRepository _stateRepository; + private readonly CveOptions _options; + private readonly CveDiagnostics _diagnostics; + private readonly TimeProvider _timeProvider; + private readonly ILogger _logger; + + public CveConnector( + SourceFetchService fetchService, + RawDocumentStorage rawDocumentStorage, + IDocumentStore documentStore, + IDtoStore dtoStore, + IAdvisoryStore advisoryStore, + ISourceStateRepository stateRepository, + IOptions options, + CveDiagnostics diagnostics, + TimeProvider? timeProvider, + ILogger logger) + { + _fetchService = fetchService ?? throw new ArgumentNullException(nameof(fetchService)); + _rawDocumentStorage = rawDocumentStorage ?? throw new ArgumentNullException(nameof(rawDocumentStorage)); + _documentStore = documentStore ?? throw new ArgumentNullException(nameof(documentStore)); + _dtoStore = dtoStore ?? throw new ArgumentNullException(nameof(dtoStore)); + _advisoryStore = advisoryStore ?? throw new ArgumentNullException(nameof(advisoryStore)); + _stateRepository = stateRepository ?? throw new ArgumentNullException(nameof(stateRepository)); + _options = (options ?? throw new ArgumentNullException(nameof(options))).Value ?? throw new ArgumentNullException(nameof(options)); + _options.Validate(); + _diagnostics = diagnostics ?? throw new ArgumentNullException(nameof(diagnostics)); + _timeProvider = timeProvider ?? TimeProvider.System; + _logger = logger ?? throw new ArgumentNullException(nameof(logger)); + } + + public string SourceName => CveConnectorPlugin.SourceName; + + public async Task FetchAsync(IServiceProvider services, CancellationToken cancellationToken) + { + ArgumentNullException.ThrowIfNull(services); + + var now = _timeProvider.GetUtcNow(); + var cursor = await GetCursorAsync(cancellationToken).ConfigureAwait(false); + + var pendingDocuments = cursor.PendingDocuments.ToHashSet(); + var pendingMappings = cursor.PendingMappings.ToHashSet(); + + var since = cursor.CurrentWindowStart ?? cursor.LastModifiedExclusive ?? now - _options.InitialBackfill; + if (since > now) + { + since = now; + } + + var windowEnd = cursor.CurrentWindowEnd ?? now; + if (windowEnd <= since) + { + windowEnd = since + TimeSpan.FromMinutes(1); + } + + var page = cursor.NextPage <= 0 ? 1 : cursor.NextPage; + var pagesFetched = 0; + var hasMorePages = true; + DateTimeOffset? maxModified = cursor.LastModifiedExclusive; + + while (hasMorePages && pagesFetched < _options.MaxPagesPerFetch) + { + cancellationToken.ThrowIfCancellationRequested(); + + var requestUri = BuildListRequestUri(since, windowEnd, page, _options.PageSize); + var metadata = new Dictionary(StringComparer.Ordinal) + { + ["since"] = since.ToString("O"), + ["until"] = windowEnd.ToString("O"), + ["page"] = page.ToString(CultureInfo.InvariantCulture), + ["pageSize"] = _options.PageSize.ToString(CultureInfo.InvariantCulture), + }; + + SourceFetchContentResult listResult; + try + { + _diagnostics.FetchAttempt(); + listResult = await _fetchService.FetchContentAsync( + new SourceFetchRequest( + CveOptions.HttpClientName, + SourceName, + requestUri) + { + Metadata = metadata, + AcceptHeaders = new[] { "application/json" }, + }, + cancellationToken).ConfigureAwait(false); + } + catch (HttpRequestException ex) + { + _diagnostics.FetchFailure(); + await _stateRepository.MarkFailureAsync(SourceName, now, _options.FailureBackoff, ex.Message, cancellationToken).ConfigureAwait(false); + throw; + } + + if (listResult.IsNotModified) + { + _diagnostics.FetchUnchanged(); + break; + } + + if (!listResult.IsSuccess || listResult.Content is null) + { + _diagnostics.FetchFailure(); + break; + } + + var pageModel = CveListParser.Parse(listResult.Content, page, _options.PageSize); + + if (pageModel.Items.Count == 0) + { + hasMorePages = false; + } + + foreach (var item in pageModel.Items) + { + cancellationToken.ThrowIfCancellationRequested(); + + var detailUri = BuildDetailRequestUri(item.CveId); + var detailMetadata = new Dictionary(StringComparer.Ordinal) + { + ["cveId"] = item.CveId, + ["page"] = page.ToString(CultureInfo.InvariantCulture), + ["since"] = since.ToString("O"), + ["until"] = windowEnd.ToString("O"), + }; + + SourceFetchResult detailResult; + try + { + detailResult = await _fetchService.FetchAsync( + new SourceFetchRequest( + CveOptions.HttpClientName, + SourceName, + detailUri) + { + Metadata = detailMetadata, + AcceptHeaders = new[] { "application/json" }, + }, + cancellationToken).ConfigureAwait(false); + } + catch (HttpRequestException ex) + { + _diagnostics.FetchFailure(); + _logger.LogWarning(ex, "Failed fetching CVE record {CveId}", item.CveId); + continue; + } + + if (detailResult.IsNotModified) + { + _diagnostics.FetchUnchanged(); + continue; + } + + if (!detailResult.IsSuccess || detailResult.Document is null) + { + _diagnostics.FetchFailure(); + continue; + } + + _diagnostics.FetchDocument(); + pendingDocuments.Add(detailResult.Document.Id); + pendingMappings.Add(detailResult.Document.Id); + } + + if (pageModel.MaxModified.HasValue) + { + if (!maxModified.HasValue || pageModel.MaxModified > maxModified) + { + maxModified = pageModel.MaxModified; + } + } + + hasMorePages = pageModel.HasMorePages; + page = pageModel.NextPageCandidate; + pagesFetched++; + + if (hasMorePages && _options.RequestDelay > TimeSpan.Zero) + { + await Task.Delay(_options.RequestDelay, cancellationToken).ConfigureAwait(false); + } + } + + var updatedCursor = cursor + .WithPendingDocuments(pendingDocuments) + .WithPendingMappings(pendingMappings); + + if (hasMorePages) + { + updatedCursor = updatedCursor + .WithCurrentWindowStart(since) + .WithCurrentWindowEnd(windowEnd) + .WithNextPage(page); + } + else + { + var nextSince = maxModified ?? windowEnd; + updatedCursor = updatedCursor + .WithLastModifiedExclusive(nextSince) + .WithCurrentWindowStart(null) + .WithCurrentWindowEnd(null) + .WithNextPage(1); + } + + await UpdateCursorAsync(updatedCursor, cancellationToken).ConfigureAwait(false); + } + + public async Task ParseAsync(IServiceProvider services, CancellationToken cancellationToken) + { + ArgumentNullException.ThrowIfNull(services); + + var cursor = await GetCursorAsync(cancellationToken).ConfigureAwait(false); + if (cursor.PendingDocuments.Count == 0) + { + return; + } + + var remainingDocuments = cursor.PendingDocuments.ToList(); + + foreach (var documentId in cursor.PendingDocuments) + { + cancellationToken.ThrowIfCancellationRequested(); + + var document = await _documentStore.FindAsync(documentId, cancellationToken).ConfigureAwait(false); + if (document is null) + { + remainingDocuments.Remove(documentId); + continue; + } + + if (!document.GridFsId.HasValue) + { + _diagnostics.ParseFailure(); + _logger.LogWarning("CVEs document {DocumentId} missing GridFS content", documentId); + await _documentStore.UpdateStatusAsync(document.Id, DocumentStatuses.Failed, cancellationToken).ConfigureAwait(false); + remainingDocuments.Remove(documentId); + continue; + } + + byte[] rawBytes; + try + { + rawBytes = await _rawDocumentStorage.DownloadAsync(document.GridFsId.Value, cancellationToken).ConfigureAwait(false); + } + catch (Exception ex) + { + _diagnostics.ParseFailure(); + _logger.LogError(ex, "Unable to download CVE raw document {DocumentId}", documentId); + throw; + } + + CveRecordDto dto; + try + { + dto = CveRecordParser.Parse(rawBytes); + } + catch (JsonException ex) + { + _diagnostics.ParseQuarantine(); + _logger.LogError(ex, "Malformed CVE JSON for {DocumentId}", documentId); + await _documentStore.UpdateStatusAsync(document.Id, DocumentStatuses.Failed, cancellationToken).ConfigureAwait(false); + remainingDocuments.Remove(documentId); + continue; + } + + var payload = BsonDocument.Parse(JsonSerializer.Serialize(dto, SerializerOptions)); + var dtoRecord = new DtoRecord( + Guid.NewGuid(), + document.Id, + SourceName, + "cve/5.0", + payload, + _timeProvider.GetUtcNow()); + + await _dtoStore.UpsertAsync(dtoRecord, cancellationToken).ConfigureAwait(false); + await _documentStore.UpdateStatusAsync(document.Id, DocumentStatuses.PendingMap, cancellationToken).ConfigureAwait(false); + + remainingDocuments.Remove(documentId); + _diagnostics.ParseSuccess(); + } + + var updatedCursor = cursor + .WithPendingDocuments(remainingDocuments); + + await UpdateCursorAsync(updatedCursor, cancellationToken).ConfigureAwait(false); + } + + public async Task MapAsync(IServiceProvider services, CancellationToken cancellationToken) + { + ArgumentNullException.ThrowIfNull(services); + + var cursor = await GetCursorAsync(cancellationToken).ConfigureAwait(false); + if (cursor.PendingMappings.Count == 0) + { + return; + } + + var pendingMappings = cursor.PendingMappings.ToList(); + + foreach (var documentId in cursor.PendingMappings) + { + cancellationToken.ThrowIfCancellationRequested(); + + var dtoRecord = await _dtoStore.FindByDocumentIdAsync(documentId, cancellationToken).ConfigureAwait(false); + var document = await _documentStore.FindAsync(documentId, cancellationToken).ConfigureAwait(false); + + if (dtoRecord is null || document is null) + { + _logger.LogWarning("Skipping CVE mapping for {DocumentId}: DTO or document missing", documentId); + pendingMappings.Remove(documentId); + continue; + } + + CveRecordDto dto; + try + { + dto = JsonSerializer.Deserialize(dtoRecord.Payload.ToJson(), SerializerOptions) + ?? throw new InvalidOperationException("Deserialized DTO was null."); + } + catch (Exception ex) + { + _logger.LogError(ex, "Failed to deserialize CVE DTO for {DocumentId}", documentId); + await _documentStore.UpdateStatusAsync(document.Id, DocumentStatuses.Failed, cancellationToken).ConfigureAwait(false); + pendingMappings.Remove(documentId); + continue; + } + + var recordedAt = dtoRecord.ValidatedAt; + var advisory = CveMapper.Map(dto, document, recordedAt); + + await _advisoryStore.UpsertAsync(advisory, cancellationToken).ConfigureAwait(false); + await _documentStore.UpdateStatusAsync(document.Id, DocumentStatuses.Mapped, cancellationToken).ConfigureAwait(false); + pendingMappings.Remove(documentId); + _diagnostics.MapSuccess(1); + } + + var updatedCursor = cursor.WithPendingMappings(pendingMappings); + await UpdateCursorAsync(updatedCursor, cancellationToken).ConfigureAwait(false); + } + + private async Task GetCursorAsync(CancellationToken cancellationToken) + { + var state = await _stateRepository.TryGetAsync(SourceName, cancellationToken).ConfigureAwait(false); + return state is null ? CveCursor.Empty : CveCursor.FromBson(state.Cursor); + } + + private async Task UpdateCursorAsync(CveCursor cursor, CancellationToken cancellationToken) + { + await _stateRepository.UpdateCursorAsync(SourceName, cursor.ToBsonDocument(), _timeProvider.GetUtcNow(), cancellationToken).ConfigureAwait(false); + } + + private static Uri BuildListRequestUri(DateTimeOffset since, DateTimeOffset until, int page, int pageSize) + { + var query = $"time_modified.gte={Uri.EscapeDataString(since.ToString("O"))}&time_modified.lte={Uri.EscapeDataString(until.ToString("O"))}&page={page}&size={pageSize}"; + return new Uri($"cve?{query}", UriKind.Relative); + } + + private static Uri BuildDetailRequestUri(string cveId) + { + var encoded = Uri.EscapeDataString(cveId); + return new Uri($"cve/{encoded}", UriKind.Relative); + } +} diff --git a/src/StellaOps.Feedser.Source.Cve/CveConnectorPlugin.cs b/src/StellaOps.Feedser.Source.Cve/CveConnectorPlugin.cs new file mode 100644 index 00000000..1e534726 --- /dev/null +++ b/src/StellaOps.Feedser.Source.Cve/CveConnectorPlugin.cs @@ -0,0 +1,19 @@ +using Microsoft.Extensions.DependencyInjection; +using StellaOps.Plugin; + +namespace StellaOps.Feedser.Source.Cve; + +public sealed class CveConnectorPlugin : IConnectorPlugin +{ + public const string SourceName = "cve"; + + public string Name => SourceName; + + public bool IsAvailable(IServiceProvider services) => services is not null; + + public IFeedConnector Create(IServiceProvider services) + { + ArgumentNullException.ThrowIfNull(services); + return ActivatorUtilities.CreateInstance(services); + } +} diff --git a/src/StellaOps.Feedser.Source.Cve/CveDependencyInjectionRoutine.cs b/src/StellaOps.Feedser.Source.Cve/CveDependencyInjectionRoutine.cs new file mode 100644 index 00000000..519bf529 --- /dev/null +++ b/src/StellaOps.Feedser.Source.Cve/CveDependencyInjectionRoutine.cs @@ -0,0 +1,54 @@ +using System; +using Microsoft.Extensions.Configuration; +using Microsoft.Extensions.DependencyInjection; +using StellaOps.DependencyInjection; +using StellaOps.Feedser.Core.Jobs; +using StellaOps.Feedser.Source.Cve.Configuration; + +namespace StellaOps.Feedser.Source.Cve; + +public sealed class CveDependencyInjectionRoutine : IDependencyInjectionRoutine +{ + private const string ConfigurationSection = "feedser:sources:cve"; + + public IServiceCollection Register(IServiceCollection services, IConfiguration configuration) + { + ArgumentNullException.ThrowIfNull(services); + ArgumentNullException.ThrowIfNull(configuration); + + services.AddCveConnector(options => + { + configuration.GetSection(ConfigurationSection).Bind(options); + options.Validate(); + }); + + services.AddTransient(); + services.AddTransient(); + services.AddTransient(); + + services.PostConfigure(options => + { + EnsureJob(options, CveJobKinds.Fetch, typeof(CveFetchJob)); + EnsureJob(options, CveJobKinds.Parse, typeof(CveParseJob)); + EnsureJob(options, CveJobKinds.Map, typeof(CveMapJob)); + }); + + return services; + } + + private static void EnsureJob(JobSchedulerOptions options, string kind, Type jobType) + { + if (options.Definitions.ContainsKey(kind)) + { + return; + } + + options.Definitions[kind] = new JobDefinition( + kind, + jobType, + options.DefaultTimeout, + options.DefaultLeaseDuration, + CronExpression: null, + Enabled: true); + } +} diff --git a/src/StellaOps.Feedser.Source.Cve/CveServiceCollectionExtensions.cs b/src/StellaOps.Feedser.Source.Cve/CveServiceCollectionExtensions.cs new file mode 100644 index 00000000..7ca226c7 --- /dev/null +++ b/src/StellaOps.Feedser.Source.Cve/CveServiceCollectionExtensions.cs @@ -0,0 +1,38 @@ +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Options; +using StellaOps.Feedser.Source.Common.Http; +using StellaOps.Feedser.Source.Cve.Configuration; +using StellaOps.Feedser.Source.Cve.Internal; + +namespace StellaOps.Feedser.Source.Cve; + +public static class CveServiceCollectionExtensions +{ + public static IServiceCollection AddCveConnector(this IServiceCollection services, Action configure) + { + ArgumentNullException.ThrowIfNull(services); + ArgumentNullException.ThrowIfNull(configure); + + services.AddOptions() + .Configure(configure) + .PostConfigure(static opts => opts.Validate()); + + services.AddSourceHttpClient(CveOptions.HttpClientName, (sp, clientOptions) => + { + var options = sp.GetRequiredService>().Value; + clientOptions.BaseAddress = options.BaseEndpoint; + clientOptions.Timeout = TimeSpan.FromSeconds(30); + clientOptions.UserAgent = "StellaOps.Feedser.Cve/1.0"; + clientOptions.AllowedHosts.Clear(); + clientOptions.AllowedHosts.Add(options.BaseEndpoint.Host); + clientOptions.DefaultRequestHeaders["Accept"] = "application/json"; + clientOptions.DefaultRequestHeaders["CVE-API-ORG"] = options.ApiOrg; + clientOptions.DefaultRequestHeaders["CVE-API-USER"] = options.ApiUser; + clientOptions.DefaultRequestHeaders["CVE-API-KEY"] = options.ApiKey; + }); + + services.AddSingleton(); + services.AddTransient(); + return services; + } +} diff --git a/src/StellaOps.Feedser.Source.Cve/Internal/CveCursor.cs b/src/StellaOps.Feedser.Source.Cve/Internal/CveCursor.cs new file mode 100644 index 00000000..b1a3136d --- /dev/null +++ b/src/StellaOps.Feedser.Source.Cve/Internal/CveCursor.cs @@ -0,0 +1,135 @@ +using System.Collections.Generic; +using System.Linq; +using MongoDB.Bson; + +namespace StellaOps.Feedser.Source.Cve.Internal; + +internal sealed record CveCursor( + DateTimeOffset? LastModifiedExclusive, + DateTimeOffset? CurrentWindowStart, + DateTimeOffset? CurrentWindowEnd, + int NextPage, + IReadOnlyCollection PendingDocuments, + IReadOnlyCollection PendingMappings) +{ + private static readonly IReadOnlyCollection EmptyGuidList = Array.Empty(); + + public static CveCursor Empty { get; } = new( + LastModifiedExclusive: null, + CurrentWindowStart: null, + CurrentWindowEnd: null, + NextPage: 1, + PendingDocuments: EmptyGuidList, + PendingMappings: EmptyGuidList); + + public BsonDocument ToBsonDocument() + { + var document = new BsonDocument + { + ["nextPage"] = NextPage, + ["pendingDocuments"] = new BsonArray(PendingDocuments.Select(id => id.ToString())), + ["pendingMappings"] = new BsonArray(PendingMappings.Select(id => id.ToString())), + }; + + if (LastModifiedExclusive.HasValue) + { + document["lastModifiedExclusive"] = LastModifiedExclusive.Value.UtcDateTime; + } + + if (CurrentWindowStart.HasValue) + { + document["currentWindowStart"] = CurrentWindowStart.Value.UtcDateTime; + } + + if (CurrentWindowEnd.HasValue) + { + document["currentWindowEnd"] = CurrentWindowEnd.Value.UtcDateTime; + } + + return document; + } + + public static CveCursor FromBson(BsonDocument? document) + { + if (document is null || document.ElementCount == 0) + { + return Empty; + } + + var lastModifiedExclusive = document.TryGetValue("lastModifiedExclusive", out var lastModifiedValue) + ? ParseDate(lastModifiedValue) + : null; + var currentWindowStart = document.TryGetValue("currentWindowStart", out var windowStartValue) + ? ParseDate(windowStartValue) + : null; + var currentWindowEnd = document.TryGetValue("currentWindowEnd", out var windowEndValue) + ? ParseDate(windowEndValue) + : null; + var nextPage = document.TryGetValue("nextPage", out var nextPageValue) && nextPageValue.IsInt32 + ? Math.Max(1, nextPageValue.AsInt32) + : 1; + + var pendingDocuments = ReadGuidArray(document, "pendingDocuments"); + var pendingMappings = ReadGuidArray(document, "pendingMappings"); + + return new CveCursor( + LastModifiedExclusive: lastModifiedExclusive, + CurrentWindowStart: currentWindowStart, + CurrentWindowEnd: currentWindowEnd, + NextPage: nextPage, + PendingDocuments: pendingDocuments, + PendingMappings: pendingMappings); + } + + public CveCursor WithPendingDocuments(IEnumerable ids) + => this with { PendingDocuments = ids?.Distinct().ToArray() ?? EmptyGuidList }; + + public CveCursor WithPendingMappings(IEnumerable ids) + => this with { PendingMappings = ids?.Distinct().ToArray() ?? EmptyGuidList }; + + public CveCursor WithLastModifiedExclusive(DateTimeOffset? timestamp) + => this with { LastModifiedExclusive = timestamp }; + + public CveCursor WithCurrentWindowEnd(DateTimeOffset? timestamp) + => this with { CurrentWindowEnd = timestamp }; + + public CveCursor WithCurrentWindowStart(DateTimeOffset? timestamp) + => this with { CurrentWindowStart = timestamp }; + + public CveCursor WithNextPage(int page) + => this with { NextPage = page < 1 ? 1 : page }; + + private static DateTimeOffset? ParseDate(BsonValue value) + { + return value.BsonType switch + { + BsonType.DateTime => DateTime.SpecifyKind(value.ToUniversalTime(), DateTimeKind.Utc), + BsonType.String when DateTimeOffset.TryParse(value.AsString, out var parsed) => parsed.ToUniversalTime(), + _ => null, + }; + } + + private static IReadOnlyCollection ReadGuidArray(BsonDocument document, string field) + { + if (!document.TryGetValue(field, out var value) || value is not BsonArray array) + { + return EmptyGuidList; + } + + var results = new List(array.Count); + foreach (var element in array) + { + if (element is null) + { + continue; + } + + if (Guid.TryParse(element.ToString(), out var guid)) + { + results.Add(guid); + } + } + + return results; + } +} diff --git a/src/StellaOps.Feedser.Source.Cve/Internal/CveDiagnostics.cs b/src/StellaOps.Feedser.Source.Cve/Internal/CveDiagnostics.cs new file mode 100644 index 00000000..6f1d73a4 --- /dev/null +++ b/src/StellaOps.Feedser.Source.Cve/Internal/CveDiagnostics.cs @@ -0,0 +1,74 @@ +using System.Diagnostics.Metrics; + +namespace StellaOps.Feedser.Source.Cve.Internal; + +public sealed class CveDiagnostics : IDisposable +{ + public const string MeterName = "StellaOps.Feedser.Source.Cve"; + public const string MeterVersion = "1.0.0"; + + private readonly Meter _meter; + private readonly Counter _fetchAttempts; + private readonly Counter _fetchDocuments; + private readonly Counter _fetchFailures; + private readonly Counter _fetchUnchanged; + private readonly Counter _parseSuccess; + private readonly Counter _parseFailures; + private readonly Counter _parseQuarantine; + private readonly Counter _mapSuccess; + + public CveDiagnostics() + { + _meter = new Meter(MeterName, MeterVersion); + _fetchAttempts = _meter.CreateCounter( + name: "cve.fetch.attempts", + unit: "operations", + description: "Number of CVE fetch operations attempted."); + _fetchDocuments = _meter.CreateCounter( + name: "cve.fetch.documents", + unit: "documents", + description: "Count of CVE documents fetched and persisted."); + _fetchFailures = _meter.CreateCounter( + name: "cve.fetch.failures", + unit: "operations", + description: "Count of CVE fetch attempts that resulted in an error."); + _fetchUnchanged = _meter.CreateCounter( + name: "cve.fetch.unchanged", + unit: "operations", + description: "Count of CVE fetch attempts returning 304 Not Modified."); + _parseSuccess = _meter.CreateCounter( + name: "cve.parse.success", + unit: "documents", + description: "Count of CVE documents successfully parsed into DTOs."); + _parseFailures = _meter.CreateCounter( + name: "cve.parse.failures", + unit: "documents", + description: "Count of CVE documents that could not be parsed."); + _parseQuarantine = _meter.CreateCounter( + name: "cve.parse.quarantine", + unit: "documents", + description: "Count of CVE documents quarantined after schema validation errors."); + _mapSuccess = _meter.CreateCounter( + name: "cve.map.success", + unit: "advisories", + description: "Count of canonical advisories emitted by the CVE mapper."); + } + + public void FetchAttempt() => _fetchAttempts.Add(1); + + public void FetchDocument() => _fetchDocuments.Add(1); + + public void FetchFailure() => _fetchFailures.Add(1); + + public void FetchUnchanged() => _fetchUnchanged.Add(1); + + public void ParseSuccess() => _parseSuccess.Add(1); + + public void ParseFailure() => _parseFailures.Add(1); + + public void ParseQuarantine() => _parseQuarantine.Add(1); + + public void MapSuccess(long count) => _mapSuccess.Add(count); + + public void Dispose() => _meter.Dispose(); +} diff --git a/src/StellaOps.Feedser.Source.Cve/Internal/CveListParser.cs b/src/StellaOps.Feedser.Source.Cve/Internal/CveListParser.cs new file mode 100644 index 00000000..93c3141e --- /dev/null +++ b/src/StellaOps.Feedser.Source.Cve/Internal/CveListParser.cs @@ -0,0 +1,264 @@ +using System.Globalization; +using System.Text.Json; + +namespace StellaOps.Feedser.Source.Cve.Internal; + +internal static class CveListParser +{ + public static CveListPage Parse(ReadOnlySpan content, int currentPage, int pageSize) + { + using var document = JsonDocument.Parse(content.ToArray()); + var root = document.RootElement; + + var items = new List(); + DateTimeOffset? maxModified = null; + + foreach (var element in EnumerateItemElements(root)) + { + var cveId = ExtractCveId(element); + if (string.IsNullOrWhiteSpace(cveId)) + { + continue; + } + + var modified = ExtractModified(element); + if (modified.HasValue && (!maxModified.HasValue || modified > maxModified)) + { + maxModified = modified; + } + + items.Add(new CveListItem(cveId, modified)); + } + + var hasMore = TryDetermineHasMore(root, currentPage, pageSize, items.Count, out var nextPage); + + return new CveListPage(items, maxModified, hasMore, nextPage ?? currentPage + 1); + } + + private static IEnumerable EnumerateItemElements(JsonElement root) + { + if (root.TryGetProperty("data", out var dataElement) && dataElement.ValueKind == JsonValueKind.Array) + { + foreach (var item in dataElement.EnumerateArray()) + { + yield return item; + } + yield break; + } + + if (root.TryGetProperty("vulnerabilities", out var vulnerabilities) && vulnerabilities.ValueKind == JsonValueKind.Array) + { + foreach (var item in vulnerabilities.EnumerateArray()) + { + yield return item; + } + yield break; + } + + if (root.ValueKind == JsonValueKind.Array) + { + foreach (var item in root.EnumerateArray()) + { + yield return item; + } + } + } + + private static string? ExtractCveId(JsonElement element) + { + if (element.TryGetProperty("cveId", out var cveId) && cveId.ValueKind == JsonValueKind.String) + { + return cveId.GetString(); + } + + if (element.TryGetProperty("cveMetadata", out var metadata)) + { + if (metadata.TryGetProperty("cveId", out var metadataId) && metadataId.ValueKind == JsonValueKind.String) + { + return metadataId.GetString(); + } + } + + if (element.TryGetProperty("cve", out var cve) && cve.ValueKind == JsonValueKind.Object) + { + if (cve.TryGetProperty("cveMetadata", out var nestedMeta) && nestedMeta.ValueKind == JsonValueKind.Object) + { + if (nestedMeta.TryGetProperty("cveId", out var nestedId) && nestedId.ValueKind == JsonValueKind.String) + { + return nestedId.GetString(); + } + } + + if (cve.TryGetProperty("id", out var cveIdElement) && cveIdElement.ValueKind == JsonValueKind.String) + { + return cveIdElement.GetString(); + } + } + + return null; + } + + private static DateTimeOffset? ExtractModified(JsonElement element) + { + static DateTimeOffset? Parse(JsonElement candidate) + { + return candidate.ValueKind switch + { + JsonValueKind.String when DateTimeOffset.TryParse(candidate.GetString(), CultureInfo.InvariantCulture, DateTimeStyles.AssumeUniversal, out var parsed) + => parsed.ToUniversalTime(), + _ => null, + }; + } + + if (element.TryGetProperty("dateUpdated", out var dateUpdated)) + { + var parsed = Parse(dateUpdated); + if (parsed.HasValue) + { + return parsed; + } + } + + if (element.TryGetProperty("cveMetadata", out var metadata)) + { + if (metadata.TryGetProperty("dateUpdated", out var metadataUpdated)) + { + var parsed = Parse(metadataUpdated); + if (parsed.HasValue) + { + return parsed; + } + } + } + + if (element.TryGetProperty("cve", out var cve) && cve.ValueKind == JsonValueKind.Object) + { + if (cve.TryGetProperty("cveMetadata", out var nestedMeta)) + { + if (nestedMeta.TryGetProperty("dateUpdated", out var nestedUpdated)) + { + var parsed = Parse(nestedUpdated); + if (parsed.HasValue) + { + return parsed; + } + } + } + + if (cve.TryGetProperty("lastModified", out var lastModified)) + { + var parsed = Parse(lastModified); + if (parsed.HasValue) + { + return parsed; + } + } + } + + return null; + } + + private static bool TryDetermineHasMore(JsonElement root, int currentPage, int pageSize, int itemCount, out int? nextPage) + { + nextPage = null; + + if (root.TryGetProperty("pagination", out var pagination) && pagination.ValueKind == JsonValueKind.Object) + { + var totalPages = TryGetInt(pagination, "totalPages") + ?? TryGetInt(pagination, "pageCount") + ?? TryGetInt(pagination, "totalPagesCount"); + if (totalPages.HasValue) + { + if (currentPage < totalPages.Value) + { + nextPage = currentPage + 1; + return true; + } + + return false; + } + + var totalCount = TryGetInt(pagination, "totalCount") + ?? TryGetInt(pagination, "totalResults"); + var limit = TryGetInt(pagination, "limit") + ?? TryGetInt(pagination, "itemsPerPage") + ?? TryGetInt(pagination, "pageSize") + ?? pageSize; + + if (totalCount.HasValue) + { + var processed = (currentPage - 1) * limit + itemCount; + if (processed < totalCount.Value) + { + nextPage = currentPage + 1; + return true; + } + + return false; + } + + if (pagination.TryGetProperty("nextPage", out var nextPageElement)) + { + switch (nextPageElement.ValueKind) + { + case JsonValueKind.Number when nextPageElement.TryGetInt32(out var value): + nextPage = value; + return true; + case JsonValueKind.String when int.TryParse(nextPageElement.GetString(), out var parsed): + nextPage = parsed; + return true; + case JsonValueKind.String when !string.IsNullOrWhiteSpace(nextPageElement.GetString()): + nextPage = currentPage + 1; + return true; + } + } + } + + if (root.TryGetProperty("nextPage", out var nextPageValue)) + { + switch (nextPageValue.ValueKind) + { + case JsonValueKind.Number when nextPageValue.TryGetInt32(out var value): + nextPage = value; + return true; + case JsonValueKind.String when int.TryParse(nextPageValue.GetString(), out var parsed): + nextPage = parsed; + return true; + case JsonValueKind.String when !string.IsNullOrWhiteSpace(nextPageValue.GetString()): + nextPage = currentPage + 1; + return true; + } + } + + if (itemCount >= pageSize) + { + nextPage = currentPage + 1; + return true; + } + + return false; + } + + private static int? TryGetInt(JsonElement element, string propertyName) + { + if (!element.TryGetProperty(propertyName, out var value)) + { + return null; + } + + return value.ValueKind switch + { + JsonValueKind.Number when value.TryGetInt32(out var number) => number, + JsonValueKind.String when int.TryParse(value.GetString(), NumberStyles.Integer, CultureInfo.InvariantCulture, out var parsed) => parsed, + _ => null, + }; + } +} + +internal sealed record CveListPage( + IReadOnlyList Items, + DateTimeOffset? MaxModified, + bool HasMorePages, + int NextPageCandidate); + +internal sealed record CveListItem(string CveId, DateTimeOffset? DateUpdated); diff --git a/src/StellaOps.Feedser.Source.Cve/Internal/CveMapper.cs b/src/StellaOps.Feedser.Source.Cve/Internal/CveMapper.cs new file mode 100644 index 00000000..c566bd20 --- /dev/null +++ b/src/StellaOps.Feedser.Source.Cve/Internal/CveMapper.cs @@ -0,0 +1,278 @@ +using System.Collections.Generic; +using System.Linq; +using StellaOps.Feedser.Models; +using StellaOps.Feedser.Normalization.Cvss; +using StellaOps.Feedser.Storage.Mongo.Documents; + +namespace StellaOps.Feedser.Source.Cve.Internal; + +internal static class CveMapper +{ + private static readonly string[] SeverityOrder = + { + "critical", + "high", + "medium", + "low", + "informational", + "none", + "unknown", + }; + + public static Advisory Map(CveRecordDto dto, DocumentRecord document, DateTimeOffset recordedAt) + { + ArgumentNullException.ThrowIfNull(dto); + ArgumentNullException.ThrowIfNull(document); + + var fetchProvenance = new AdvisoryProvenance(CveConnectorPlugin.SourceName, "document", document.Uri, document.FetchedAt); + var mapProvenance = new AdvisoryProvenance(CveConnectorPlugin.SourceName, "mapping", dto.CveId, recordedAt); + + var aliases = dto.Aliases + .Append(dto.CveId) + .Where(static alias => !string.IsNullOrWhiteSpace(alias)) + .Distinct(StringComparer.OrdinalIgnoreCase) + .ToArray(); + + var references = dto.References + .Select(reference => CreateReference(reference, recordedAt)) + .Where(static reference => reference is not null) + .Cast() + .ToList(); + + var affected = CreateAffectedPackages(dto, recordedAt); + var cvssMetrics = CreateCvssMetrics(dto, recordedAt, document.Uri); + var severity = DetermineSeverity(cvssMetrics); + + var provenance = new[] + { + fetchProvenance, + mapProvenance, + }; + + var title = string.IsNullOrWhiteSpace(dto.Title) ? dto.CveId : dto.Title!; + + return new Advisory( + advisoryKey: dto.CveId, + title: title, + summary: dto.Summary, + language: dto.Language, + published: dto.Published, + modified: dto.Modified ?? dto.Published, + severity: severity, + exploitKnown: false, + aliases: aliases, + references: references, + affectedPackages: affected, + cvssMetrics: cvssMetrics, + provenance: provenance); + } + + private static AdvisoryReference? CreateReference(CveReferenceDto dto, DateTimeOffset recordedAt) + { + if (string.IsNullOrWhiteSpace(dto.Url) || !Validation.LooksLikeHttpUrl(dto.Url)) + { + return null; + } + + var kind = dto.Tags.FirstOrDefault(); + return new AdvisoryReference( + dto.Url, + kind, + dto.Source, + summary: null, + provenance: new AdvisoryProvenance(CveConnectorPlugin.SourceName, "reference", dto.Url, recordedAt)); + } + + private static IReadOnlyList CreateAffectedPackages(CveRecordDto dto, DateTimeOffset recordedAt) + { + if (dto.Affected.Count == 0) + { + return Array.Empty(); + } + + var packages = new List(dto.Affected.Count); + foreach (var affected in dto.Affected) + { + var vendor = string.IsNullOrWhiteSpace(affected.Vendor) ? "unknown-vendor" : affected.Vendor!.Trim(); + var product = string.IsNullOrWhiteSpace(affected.Product) ? "unknown-product" : affected.Product!.Trim(); + var identifier = string.Equals(product, vendor, StringComparison.OrdinalIgnoreCase) + ? vendor.ToLowerInvariant() + : $"{vendor}:{product}".ToLowerInvariant(); + + var provenance = new[] + { + new AdvisoryProvenance(CveConnectorPlugin.SourceName, "affected", identifier, recordedAt), + }; + + var primitivesDictionary = new Dictionary(StringComparer.OrdinalIgnoreCase) + { + ["vendor"] = vendor, + ["product"] = product, + }; + if (!string.IsNullOrWhiteSpace(affected.Platform)) + { + primitivesDictionary["platform"] = affected.Platform!; + } + + var primitives = new RangePrimitives(null, null, null, primitivesDictionary); + var ranges = CreateVersionRanges(affected, recordedAt, identifier, primitives); + var statuses = CreateStatuses(affected, recordedAt, identifier); + + if (ranges.Count == 0) + { + ranges.Add(new AffectedVersionRange( + rangeKind: "vendor", + introducedVersion: null, + fixedVersion: null, + lastAffectedVersion: null, + rangeExpression: null, + provenance: provenance[0], + primitives: primitives)); + } + + packages.Add(new AffectedPackage( + type: AffectedPackageTypes.Vendor, + identifier: identifier, + platform: affected.Platform, + versionRanges: ranges, + statuses: statuses, + provenance: provenance)); + } + + return packages; + } + + private static List CreateVersionRanges(CveAffectedDto affected, DateTimeOffset recordedAt, string identifier, RangePrimitives primitives) + { + var ranges = new List(); + foreach (var version in affected.Versions) + { + var provenance = new AdvisoryProvenance( + CveConnectorPlugin.SourceName, + "affected-range", + identifier, + recordedAt); + + var kind = !string.IsNullOrWhiteSpace(version.VersionType) + ? version.VersionType!.Trim().ToLowerInvariant() + : "vendor"; + + var rangeExpression = string.IsNullOrWhiteSpace(version.Range) + ? BuildRangeExpression(version) + : version.Range; + + ranges.Add(new AffectedVersionRange( + rangeKind: kind, + introducedVersion: Normalize(version.Version), + fixedVersion: Normalize(version.LessThan), + lastAffectedVersion: Normalize(version.LessThanOrEqual), + rangeExpression: Normalize(rangeExpression), + provenance: provenance, + primitives: primitives)); + } + + return ranges; + } + + private static List CreateStatuses(CveAffectedDto affected, DateTimeOffset recordedAt, string identifier) + { + var statuses = new List(); + + void AddStatus(string? status) + { + if (string.IsNullOrWhiteSpace(status)) + { + return; + } + + statuses.Add(new AffectedPackageStatus( + status, + new AdvisoryProvenance(CveConnectorPlugin.SourceName, "affected-status", identifier, recordedAt))); + } + + AddStatus(affected.DefaultStatus); + + foreach (var version in affected.Versions) + { + AddStatus(version.Status); + } + + return statuses; + } + + private static string? Normalize(string? value) + => string.IsNullOrWhiteSpace(value) || value is "*" or "-" ? null : value.Trim(); + + private static string? BuildRangeExpression(CveVersionDto version) + { + var builder = new List(); + if (!string.IsNullOrWhiteSpace(version.Version)) + { + builder.Add($"version={version.Version}"); + } + + if (!string.IsNullOrWhiteSpace(version.LessThan)) + { + builder.Add($"< {version.LessThan}"); + } + + if (!string.IsNullOrWhiteSpace(version.LessThanOrEqual)) + { + builder.Add($"<= {version.LessThanOrEqual}"); + } + + if (builder.Count == 0) + { + return null; + } + + return string.Join(", ", builder); + } + + private static IReadOnlyList CreateCvssMetrics(CveRecordDto dto, DateTimeOffset recordedAt, string sourceUri) + { + if (dto.Metrics.Count == 0) + { + return Array.Empty(); + } + + var provenance = new AdvisoryProvenance(CveConnectorPlugin.SourceName, "cvss", sourceUri, recordedAt); + var metrics = new List(dto.Metrics.Count); + foreach (var metric in dto.Metrics) + { + if (!CvssMetricNormalizer.TryNormalize(metric.Version, metric.Vector, metric.BaseScore, metric.BaseSeverity, out var normalized)) + { + continue; + } + + metrics.Add(new CvssMetric( + normalized.Version, + normalized.Vector, + normalized.BaseScore, + normalized.BaseSeverity, + provenance)); + } + + return metrics; + } + + private static string? DetermineSeverity(IReadOnlyList metrics) + { + if (metrics.Count == 0) + { + return null; + } + + foreach (var level in SeverityOrder) + { + if (metrics.Any(metric => string.Equals(metric.BaseSeverity, level, StringComparison.OrdinalIgnoreCase))) + { + return level; + } + } + + return metrics + .Select(metric => metric.BaseSeverity) + .FirstOrDefault(); + } +} diff --git a/src/StellaOps.Feedser.Source.Cve/Internal/CveRecordDto.cs b/src/StellaOps.Feedser.Source.Cve/Internal/CveRecordDto.cs new file mode 100644 index 00000000..8dbe9d54 --- /dev/null +++ b/src/StellaOps.Feedser.Source.Cve/Internal/CveRecordDto.cs @@ -0,0 +1,105 @@ +using System.Text.Json.Serialization; + +namespace StellaOps.Feedser.Source.Cve.Internal; + +internal sealed record CveRecordDto +{ + [JsonPropertyName("cveId")] + public string CveId { get; init; } = string.Empty; + + [JsonPropertyName("title")] + public string? Title { get; init; } + + [JsonPropertyName("summary")] + public string? Summary { get; init; } + + [JsonPropertyName("language")] + public string? Language { get; init; } + + [JsonPropertyName("state")] + public string State { get; init; } = "PUBLISHED"; + + [JsonPropertyName("published")] + public DateTimeOffset? Published { get; init; } + + [JsonPropertyName("modified")] + public DateTimeOffset? Modified { get; init; } + + [JsonPropertyName("aliases")] + public IReadOnlyList Aliases { get; init; } = Array.Empty(); + + [JsonPropertyName("references")] + public IReadOnlyList References { get; init; } = Array.Empty(); + + [JsonPropertyName("affected")] + public IReadOnlyList Affected { get; init; } = Array.Empty(); + + [JsonPropertyName("metrics")] + public IReadOnlyList Metrics { get; init; } = Array.Empty(); +} + +internal sealed record CveReferenceDto +{ + [JsonPropertyName("url")] + public string Url { get; init; } = string.Empty; + + [JsonPropertyName("source")] + public string? Source { get; init; } + + [JsonPropertyName("tags")] + public IReadOnlyList Tags { get; init; } = Array.Empty(); +} + +internal sealed record CveAffectedDto +{ + [JsonPropertyName("vendor")] + public string? Vendor { get; init; } + + [JsonPropertyName("product")] + public string? Product { get; init; } + + [JsonPropertyName("platform")] + public string? Platform { get; init; } + + [JsonPropertyName("defaultStatus")] + public string? DefaultStatus { get; init; } + + [JsonPropertyName("versions")] + public IReadOnlyList Versions { get; init; } = Array.Empty(); +} + +internal sealed record CveVersionDto +{ + [JsonPropertyName("status")] + public string? Status { get; init; } + + [JsonPropertyName("version")] + public string? Version { get; init; } + + [JsonPropertyName("lessThan")] + public string? LessThan { get; init; } + + [JsonPropertyName("lessThanOrEqual")] + public string? LessThanOrEqual { get; init; } + + [JsonPropertyName("versionType")] + public string? VersionType { get; init; } + + [JsonPropertyName("versionRange")] + public string? Range { get; init; } +} + +internal sealed record CveCvssMetricDto +{ + [JsonPropertyName("version")] + public string? Version { get; init; } + + [JsonPropertyName("vector")] + public string? Vector { get; init; } + + [JsonPropertyName("baseScore")] + public double? BaseScore { get; init; } + + [JsonPropertyName("baseSeverity")] + public string? BaseSeverity { get; init; } +} diff --git a/src/StellaOps.Feedser.Source.Cve/Internal/CveRecordParser.cs b/src/StellaOps.Feedser.Source.Cve/Internal/CveRecordParser.cs new file mode 100644 index 00000000..d9fc89d7 --- /dev/null +++ b/src/StellaOps.Feedser.Source.Cve/Internal/CveRecordParser.cs @@ -0,0 +1,346 @@ +using System.Globalization; +using System.Linq; +using System.Text.Json; +using StellaOps.Feedser.Normalization.Text; + +namespace StellaOps.Feedser.Source.Cve.Internal; + +internal static class CveRecordParser +{ + public static CveRecordDto Parse(ReadOnlySpan content) + { + using var document = JsonDocument.Parse(content.ToArray()); + var root = document.RootElement; + + var metadata = TryGetProperty(root, "cveMetadata"); + if (metadata.ValueKind != JsonValueKind.Object) + { + throw new JsonException("cveMetadata section missing."); + } + + var containers = TryGetProperty(root, "containers"); + var cna = TryGetProperty(containers, "cna"); + + var cveId = GetString(metadata, "cveId") ?? throw new JsonException("cveMetadata.cveId missing."); + var state = GetString(metadata, "state") ?? "PUBLISHED"; + var published = GetDate(metadata, "datePublished"); + var modified = GetDate(metadata, "dateUpdated") ?? GetDate(metadata, "dateReserved"); + + var description = ParseDescription(cna); + + var aliases = new HashSet(StringComparer.OrdinalIgnoreCase) + { + cveId, + }; + foreach (var alias in ParseAliases(cna)) + { + aliases.Add(alias); + } + + var references = ParseReferences(cna); + var affected = ParseAffected(cna); + var metrics = ParseMetrics(cna); + + return new CveRecordDto + { + CveId = cveId, + Title = GetString(cna, "title") ?? cveId, + Summary = description.Text, + Language = description.Language, + State = state, + Published = published, + Modified = modified, + Aliases = aliases.ToArray(), + References = references, + Affected = affected, + Metrics = metrics, + }; + } + + private static NormalizedDescription ParseDescription(JsonElement element) + { + if (element.ValueKind != JsonValueKind.Object) + { + return DescriptionNormalizer.Normalize(Array.Empty()); + } + + if (!element.TryGetProperty("descriptions", out var descriptions) || descriptions.ValueKind != JsonValueKind.Array) + { + return DescriptionNormalizer.Normalize(Array.Empty()); + } + + var items = new List(descriptions.GetArrayLength()); + foreach (var entry in descriptions.EnumerateArray()) + { + if (entry.ValueKind != JsonValueKind.Object) + { + continue; + } + + var text = GetString(entry, "value"); + if (string.IsNullOrWhiteSpace(text)) + { + continue; + } + + var lang = GetString(entry, "lang"); + items.Add(new LocalizedText(text, lang)); + } + + return DescriptionNormalizer.Normalize(items); + } + + private static IEnumerable ParseAliases(JsonElement element) + { + if (element.ValueKind != JsonValueKind.Object) + { + yield break; + } + + if (!element.TryGetProperty("aliases", out var aliases) || aliases.ValueKind != JsonValueKind.Array) + { + yield break; + } + + foreach (var alias in aliases.EnumerateArray()) + { + if (alias.ValueKind == JsonValueKind.String) + { + var value = alias.GetString(); + if (!string.IsNullOrWhiteSpace(value)) + { + yield return value; + } + } + } + } + + private static IReadOnlyList ParseReferences(JsonElement element) + { + if (element.ValueKind != JsonValueKind.Object) + { + return Array.Empty(); + } + + if (!element.TryGetProperty("references", out var references) || references.ValueKind != JsonValueKind.Array) + { + return Array.Empty(); + } + + var list = new List(references.GetArrayLength()); + foreach (var reference in references.EnumerateArray()) + { + if (reference.ValueKind != JsonValueKind.Object) + { + continue; + } + + var url = GetString(reference, "url"); + if (string.IsNullOrWhiteSpace(url)) + { + continue; + } + + var tags = Array.Empty(); + if (reference.TryGetProperty("tags", out var tagsElement) && tagsElement.ValueKind == JsonValueKind.Array) + { + tags = tagsElement + .EnumerateArray() + .Where(static t => t.ValueKind == JsonValueKind.String) + .Select(static t => t.GetString()!) + .Where(static v => !string.IsNullOrWhiteSpace(v)) + .ToArray(); + } + + var source = GetString(reference, "name") ?? GetString(reference, "source"); + list.Add(new CveReferenceDto + { + Url = url, + Source = source, + Tags = tags, + }); + } + + return list; + } + + private static IReadOnlyList ParseAffected(JsonElement element) + { + if (element.ValueKind != JsonValueKind.Object) + { + return Array.Empty(); + } + + if (!element.TryGetProperty("affected", out var affected) || affected.ValueKind != JsonValueKind.Array) + { + return Array.Empty(); + } + + var list = new List(affected.GetArrayLength()); + foreach (var item in affected.EnumerateArray()) + { + if (item.ValueKind != JsonValueKind.Object) + { + continue; + } + + var versions = new List(); + if (item.TryGetProperty("versions", out var versionsElement) && versionsElement.ValueKind == JsonValueKind.Array) + { + foreach (var versionEntry in versionsElement.EnumerateArray()) + { + if (versionEntry.ValueKind != JsonValueKind.Object) + { + continue; + } + + versions.Add(new CveVersionDto + { + Status = GetString(versionEntry, "status"), + Version = GetString(versionEntry, "version"), + LessThan = GetString(versionEntry, "lessThan"), + LessThanOrEqual = GetString(versionEntry, "lessThanOrEqual"), + VersionType = GetString(versionEntry, "versionType"), + Range = GetString(versionEntry, "versionRange"), + }); + } + } + + list.Add(new CveAffectedDto + { + Vendor = GetString(item, "vendor") ?? GetString(item, "vendorName"), + Product = GetString(item, "product") ?? GetString(item, "productName"), + Platform = GetString(item, "platform"), + DefaultStatus = GetString(item, "defaultStatus"), + Versions = versions, + }); + } + + return list; + } + + private static IReadOnlyList ParseMetrics(JsonElement element) + { + if (element.ValueKind != JsonValueKind.Object) + { + return Array.Empty(); + } + + if (!element.TryGetProperty("metrics", out var metrics) || metrics.ValueKind != JsonValueKind.Array) + { + return Array.Empty(); + } + + var list = new List(metrics.GetArrayLength()); + foreach (var metric in metrics.EnumerateArray()) + { + if (metric.ValueKind != JsonValueKind.Object) + { + continue; + } + + if (metric.TryGetProperty("cvssV4_0", out var cvss40) && cvss40.ValueKind == JsonValueKind.Object) + { + list.Add(ParseCvss(cvss40, "4.0")); + } + else if (metric.TryGetProperty("cvssV3_1", out var cvss31) && cvss31.ValueKind == JsonValueKind.Object) + { + list.Add(ParseCvss(cvss31, "3.1")); + } + else if (metric.TryGetProperty("cvssV3", out var cvss3) && cvss3.ValueKind == JsonValueKind.Object) + { + list.Add(ParseCvss(cvss3, "3.0")); + } + else if (metric.TryGetProperty("cvssV2", out var cvss2) && cvss2.ValueKind == JsonValueKind.Object) + { + list.Add(ParseCvss(cvss2, "2.0")); + } + } + + return list; + } + + private static CveCvssMetricDto ParseCvss(JsonElement element, string fallbackVersion) + { + var version = GetString(element, "version") ?? fallbackVersion; + var vector = GetString(element, "vectorString") ?? GetString(element, "vector"); + var baseScore = GetDouble(element, "baseScore"); + var severity = GetString(element, "baseSeverity") ?? GetString(element, "severity"); + + return new CveCvssMetricDto + { + Version = version, + Vector = vector, + BaseScore = baseScore, + BaseSeverity = severity, + }; + } + + private static JsonElement TryGetProperty(JsonElement element, string propertyName) + { + if (element.ValueKind == JsonValueKind.Object && element.TryGetProperty(propertyName, out var property)) + { + return property; + } + + return default; + } + + private static string? GetString(JsonElement element, string propertyName) + { + if (element.ValueKind != JsonValueKind.Object) + { + return null; + } + + if (!element.TryGetProperty(propertyName, out var property)) + { + return null; + } + + return property.ValueKind switch + { + JsonValueKind.String => property.GetString(), + JsonValueKind.Number when property.TryGetDouble(out var number) => number.ToString(CultureInfo.InvariantCulture), + _ => null, + }; + } + + private static DateTimeOffset? GetDate(JsonElement element, string propertyName) + { + var value = GetString(element, propertyName); + if (string.IsNullOrWhiteSpace(value)) + { + return null; + } + + return DateTimeOffset.TryParse(value, CultureInfo.InvariantCulture, DateTimeStyles.AssumeUniversal, out var parsed) + ? parsed.ToUniversalTime() + : null; + } + + private static double? GetDouble(JsonElement element, string propertyName) + { + if (element.ValueKind != JsonValueKind.Object) + { + return null; + } + + if (!element.TryGetProperty(propertyName, out var property)) + { + return null; + } + + if (property.ValueKind == JsonValueKind.Number && property.TryGetDouble(out var number)) + { + return number; + } + + if (property.ValueKind == JsonValueKind.String && double.TryParse(property.GetString(), NumberStyles.Float, CultureInfo.InvariantCulture, out var parsed)) + { + return parsed; + } + + return null; + } +} diff --git a/src/StellaOps.Feedser.Source.Cve/Jobs.cs b/src/StellaOps.Feedser.Source.Cve/Jobs.cs new file mode 100644 index 00000000..b413a27f --- /dev/null +++ b/src/StellaOps.Feedser.Source.Cve/Jobs.cs @@ -0,0 +1,43 @@ +using StellaOps.Feedser.Core.Jobs; + +namespace StellaOps.Feedser.Source.Cve; + +internal static class CveJobKinds +{ + public const string Fetch = "source:cve:fetch"; + public const string Parse = "source:cve:parse"; + public const string Map = "source:cve:map"; +} + +internal sealed class CveFetchJob : IJob +{ + private readonly CveConnector _connector; + + public CveFetchJob(CveConnector connector) + => _connector = connector ?? throw new ArgumentNullException(nameof(connector)); + + public Task ExecuteAsync(JobExecutionContext context, CancellationToken cancellationToken) + => _connector.FetchAsync(context.Services, cancellationToken); +} + +internal sealed class CveParseJob : IJob +{ + private readonly CveConnector _connector; + + public CveParseJob(CveConnector connector) + => _connector = connector ?? throw new ArgumentNullException(nameof(connector)); + + public Task ExecuteAsync(JobExecutionContext context, CancellationToken cancellationToken) + => _connector.ParseAsync(context.Services, cancellationToken); +} + +internal sealed class CveMapJob : IJob +{ + private readonly CveConnector _connector; + + public CveMapJob(CveConnector connector) + => _connector = connector ?? throw new ArgumentNullException(nameof(connector)); + + public Task ExecuteAsync(JobExecutionContext context, CancellationToken cancellationToken) + => _connector.MapAsync(context.Services, cancellationToken); +} diff --git a/src/StellaOps.Feedser.Source.Cve/StellaOps.Feedser.Source.Cve.csproj b/src/StellaOps.Feedser.Source.Cve/StellaOps.Feedser.Source.Cve.csproj new file mode 100644 index 00000000..f7f2c154 --- /dev/null +++ b/src/StellaOps.Feedser.Source.Cve/StellaOps.Feedser.Source.Cve.csproj @@ -0,0 +1,16 @@ + + + + net10.0 + enable + enable + + + + + + + + + + diff --git a/src/StellaOps.Feedser.Source.Cve/TASKS.md b/src/StellaOps.Feedser.Source.Cve/TASKS.md new file mode 100644 index 00000000..baf05cd6 --- /dev/null +++ b/src/StellaOps.Feedser.Source.Cve/TASKS.md @@ -0,0 +1,11 @@ +# TASKS +| Task | Owner(s) | Depends on | Notes | +|---|---|---|---| +|Define CVE data source + contract|BE-Conn-CVE|Research|**DONE (2025-10-10)** – Connector targets the CVE Services JSON 5 API with authenticated windowed queries documented in `CveOptions` (`CVE-API-*` headers, pagination semantics, failure backoff).| +|Fetch/cursor implementation|BE-Conn-CVE|Source.Common, Storage.Mongo|**DONE (2025-10-10)** – Time-window + page-aware cursor with SourceFetchService fetching list/detail pairs, resumable state persisted via `CveCursor`.| +|DTOs & parser|BE-Conn-CVE|Source.Common|**DONE (2025-10-10)** – `CveRecordParser` and DTOs capture aliases, references, metrics, vendor ranges; sanitises text and timestamps.| +|Canonical mapping & range primitives|BE-Conn-CVE|Models|**DONE (2025-10-10)** – `CveMapper` emits canonical advisories, vendor range primitives, SemVer/range statuses, references, CVSS normalization.| +|Deterministic tests & fixtures|QA|Testing|**DONE (2025-10-10)** – Added `StellaOps.Feedser.Source.Cve.Tests` harness with canned fixtures + snapshot regression covering fetch/parse/map.| +|Observability & docs|DevEx|Docs|**DONE (2025-10-10)** – Diagnostics meter (`cve.fetch.*`, etc.) wired; options/usage documented via `CveServiceCollectionExtensions`.| +|Operator rollout playbook|BE-Conn-CVE, Ops|Docs|**TODO** – Document credential provisioning (CVE-API-* headers), scheduler defaults, and initial backfill procedure in ops handbook; include quota monitoring guidance.| +|Live smoke & monitoring|QA, BE-Conn-CVE|WebService, Observability|**TODO** – Run connector against staging with real credentials, validate cursor advancement and advisory writes, and add Grafana/alert rules for CVE fetch failures + rate-limit nearing.| diff --git a/src/StellaOps.Feedser.Source.Distro.Debian.Tests/DebianConnectorTests.cs b/src/StellaOps.Feedser.Source.Distro.Debian.Tests/DebianConnectorTests.cs new file mode 100644 index 00000000..bbec302a --- /dev/null +++ b/src/StellaOps.Feedser.Source.Distro.Debian.Tests/DebianConnectorTests.cs @@ -0,0 +1,281 @@ +using System.Collections.Generic; +using System; +using System.IO; +using System.Linq; +using System.Net; +using System.Net.Http; +using System.Net.Http.Headers; +using System.Text; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Http; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Logging.Abstractions; +using Microsoft.Extensions.Options; +using Microsoft.Extensions.Time.Testing; +using MongoDB.Driver; +using StellaOps.Feedser.Models; +using StellaOps.Feedser.Source.Common; +using StellaOps.Feedser.Source.Common.Http; +using StellaOps.Feedser.Source.Common.Testing; +using StellaOps.Feedser.Source.Distro.Debian.Configuration; +using StellaOps.Feedser.Storage.Mongo; +using StellaOps.Feedser.Storage.Mongo.Advisories; +using StellaOps.Feedser.Storage.Mongo.Documents; +using StellaOps.Feedser.Storage.Mongo.Dtos; +using StellaOps.Feedser.Testing; +using Xunit; +using Xunit.Abstractions; + +namespace StellaOps.Feedser.Source.Distro.Debian.Tests; + +[Collection("mongo-fixture")] +public sealed class DebianConnectorTests : IAsyncLifetime +{ + private static readonly Uri ListUri = new("https://salsa.debian.org/security-tracker-team/security-tracker/-/raw/master/data/DSA/list"); + private static readonly Uri DetailResolved = new("https://security-tracker.debian.org/tracker/DSA-2024-123"); + private static readonly Uri DetailOpen = new("https://security-tracker.debian.org/tracker/DSA-2024-124"); + + private readonly MongoIntegrationFixture _fixture; + private readonly FakeTimeProvider _timeProvider; + private readonly CannedHttpMessageHandler _handler; + private readonly Dictionary> _fallbackFactories = new(); + private readonly ITestOutputHelper _output; + + public DebianConnectorTests(MongoIntegrationFixture fixture, ITestOutputHelper output) + { + _fixture = fixture; + _handler = new CannedHttpMessageHandler(); + _handler.SetFallback(request => + { + if (request.RequestUri is null) + { + throw new InvalidOperationException("Request URI required for fallback response."); + } + + if (_fallbackFactories.TryGetValue(request.RequestUri, out var factory)) + { + return factory(request); + } + + throw new InvalidOperationException($"No canned or fallback response registered for {request.Method} {request.RequestUri}."); + }); + _timeProvider = new FakeTimeProvider(new DateTimeOffset(2024, 9, 12, 0, 0, 0, TimeSpan.Zero)); + _output = output; + } + + [Fact] + public async Task FetchParseMap_PopulatesRangePrimitivesAndResumesWithNotModified() + { + await using var provider = await BuildServiceProviderAsync(); + + SeedInitialResponses(); + + var connector = provider.GetRequiredService(); + await connector.FetchAsync(provider, CancellationToken.None); + _timeProvider.Advance(TimeSpan.FromMinutes(1)); + await connector.ParseAsync(provider, CancellationToken.None); + await connector.MapAsync(provider, CancellationToken.None); + + var advisoryStore = provider.GetRequiredService(); + var advisories = await advisoryStore.GetRecentAsync(10, CancellationToken.None); + Assert.Equal(2, advisories.Count); + + var resolved = advisories.Single(a => a.AdvisoryKey == "DSA-2024-123"); + _output.WriteLine("Resolved aliases: " + string.Join(",", resolved.Aliases)); + var resolvedBookworm = Assert.Single(resolved.AffectedPackages, p => p.Platform == "bookworm"); + var resolvedRange = Assert.Single(resolvedBookworm.VersionRanges); + Assert.Equal("evr", resolvedRange.RangeKind); + Assert.Equal("1:1.1.1n-0+deb11u2", resolvedRange.IntroducedVersion); + Assert.Equal("1:1.1.1n-0+deb11u5", resolvedRange.FixedVersion); + Assert.NotNull(resolvedRange.Primitives); + Assert.NotNull(resolvedRange.Primitives!.Evr); + Assert.Equal(1, resolvedRange.Primitives.Evr!.Introduced!.Epoch); + Assert.Equal("1.1.1n", resolvedRange.Primitives.Evr.Introduced.UpstreamVersion); + + var open = advisories.Single(a => a.AdvisoryKey == "DSA-2024-124"); + var openBookworm = Assert.Single(open.AffectedPackages, p => p.Platform == "bookworm"); + var openRange = Assert.Single(openBookworm.VersionRanges); + Assert.Equal("evr", openRange.RangeKind); + Assert.Equal("1:1.3.1-1", openRange.IntroducedVersion); + Assert.Null(openRange.FixedVersion); + Assert.NotNull(openRange.Primitives); + Assert.NotNull(openRange.Primitives!.Evr); + + // Ensure data persisted through Mongo round-trip. + var found = await advisoryStore.FindAsync("DSA-2024-123", CancellationToken.None); + Assert.NotNull(found); + var persistedRange = Assert.Single(found!.AffectedPackages, pkg => pkg.Platform == "bookworm").VersionRanges.Single(); + Assert.NotNull(persistedRange.Primitives); + Assert.NotNull(persistedRange.Primitives!.Evr); + + // Second run should issue conditional requests and no additional parsing/mapping. + SeedNotModifiedResponses(); + await connector.FetchAsync(provider, CancellationToken.None); + _timeProvider.Advance(TimeSpan.FromMinutes(1)); + await connector.ParseAsync(provider, CancellationToken.None); + await connector.MapAsync(provider, CancellationToken.None); + + var documents = provider.GetRequiredService(); + var listDoc = await documents.FindBySourceAndUriAsync(DebianConnectorPlugin.SourceName, DetailResolved.ToString(), CancellationToken.None); + Assert.NotNull(listDoc); + + var refreshed = await advisoryStore.GetRecentAsync(10, CancellationToken.None); + Assert.Equal(2, refreshed.Count); + } + + private async Task BuildServiceProviderAsync() + { + await _fixture.Client.DropDatabaseAsync(_fixture.Database.DatabaseNamespace.DatabaseName, CancellationToken.None); + _handler.Clear(); + _fallbackFactories.Clear(); + + var services = new ServiceCollection(); + services.AddLogging(builder => builder.AddProvider(new TestOutputLoggerProvider(_output))); + services.AddSingleton(_timeProvider); + services.AddSingleton(_handler); + + services.AddMongoStorage(options => + { + options.ConnectionString = _fixture.Runner.ConnectionString; + options.DatabaseName = _fixture.Database.DatabaseNamespace.DatabaseName; + options.CommandTimeout = TimeSpan.FromSeconds(5); + }); + + services.AddSourceCommon(); + services.AddDebianConnector(options => + { + options.ListEndpoint = ListUri; + options.DetailBaseUri = new Uri("https://security-tracker.debian.org/tracker/"); + options.MaxAdvisoriesPerFetch = 10; + options.RequestDelay = TimeSpan.Zero; + }); + + services.Configure(DebianOptions.HttpClientName, builderOptions => + { + builderOptions.HttpMessageHandlerBuilderActions.Add(builder => + { + builder.PrimaryHandler = _handler; + }); + }); + + var provider = services.BuildServiceProvider(); + var bootstrapper = provider.GetRequiredService(); + await bootstrapper.InitializeAsync(CancellationToken.None); + return provider; + } + + private void SeedInitialResponses() + { + AddListResponse("debian-list.txt", "\"list-v1\""); + AddDetailResponse(DetailResolved, "debian-detail-dsa-2024-123.html", "\"detail-123\""); + AddDetailResponse(DetailOpen, "debian-detail-dsa-2024-124.html", "\"detail-124\""); + } + + private void SeedNotModifiedResponses() + { + AddNotModifiedResponse(ListUri, "\"list-v1\""); + AddNotModifiedResponse(DetailResolved, "\"detail-123\""); + AddNotModifiedResponse(DetailOpen, "\"detail-124\""); + } + + private void AddListResponse(string fixture, string etag) + { + RegisterResponseFactory(ListUri, () => + { + var response = new HttpResponseMessage(HttpStatusCode.OK) + { + Content = new StringContent(ReadFixture(fixture), Encoding.UTF8, "text/plain"), + }; + response.Headers.ETag = new EntityTagHeaderValue(etag); + return response; + }); + } + + private void AddDetailResponse(Uri uri, string fixture, string etag) + { + RegisterResponseFactory(uri, () => + { + var response = new HttpResponseMessage(HttpStatusCode.OK) + { + Content = new StringContent(ReadFixture(fixture), Encoding.UTF8, "text/html"), + }; + response.Headers.ETag = new EntityTagHeaderValue(etag); + return response; + }); + } + + private void AddNotModifiedResponse(Uri uri, string etag) + { + RegisterResponseFactory(uri, () => + { + var response = new HttpResponseMessage(HttpStatusCode.NotModified); + response.Headers.ETag = new EntityTagHeaderValue(etag); + return response; + }); + } + + private void RegisterResponseFactory(Uri uri, Func factory) + { + _handler.AddResponse(uri, () => factory()); + _fallbackFactories[uri] = _ => factory(); + } + + private static string ReadFixture(string filename) + { + var candidates = new[] + { + Path.Combine(AppContext.BaseDirectory, "Source", "Distro", "Debian", "Fixtures", filename), + Path.Combine(AppContext.BaseDirectory, "Distro", "Debian", "Fixtures", filename), + Path.Combine(AppContext.BaseDirectory, "..", "..", "..", "Source", "Distro", "Debian", "Fixtures", filename), + }; + + foreach (var candidate in candidates) + { + var fullPath = Path.GetFullPath(candidate); + if (File.Exists(fullPath)) + { + return File.ReadAllText(fullPath); + } + } + + throw new FileNotFoundException($"Fixture '{filename}' not found", filename); + } + + public Task InitializeAsync() => Task.CompletedTask; + + public Task DisposeAsync() => Task.CompletedTask; + + private sealed class TestOutputLoggerProvider : ILoggerProvider + { + private readonly ITestOutputHelper _output; + + public TestOutputLoggerProvider(ITestOutputHelper output) => _output = output; + + public ILogger CreateLogger(string categoryName) => new TestOutputLogger(_output); + + public void Dispose() + { + } + + private sealed class TestOutputLogger : ILogger + { + private readonly ITestOutputHelper _output; + + public TestOutputLogger(ITestOutputHelper output) => _output = output; + + public IDisposable BeginScope(TState state) where TState : notnull => NullLogger.Instance.BeginScope(state); + + public bool IsEnabled(LogLevel logLevel) => false; + + public void Log(LogLevel logLevel, EventId eventId, TState state, Exception? exception, Func formatter) + { + if (IsEnabled(logLevel)) + { + _output.WriteLine(formatter(state, exception)); + } + } + } + } +} diff --git a/src/StellaOps.Feedser.Source.Distro.Debian.Tests/DebianMapperTests.cs b/src/StellaOps.Feedser.Source.Distro.Debian.Tests/DebianMapperTests.cs new file mode 100644 index 00000000..199b24ba --- /dev/null +++ b/src/StellaOps.Feedser.Source.Distro.Debian.Tests/DebianMapperTests.cs @@ -0,0 +1,88 @@ +using System; +using Xunit; +using StellaOps.Feedser.Models; +using StellaOps.Feedser.Source.Distro.Debian; +using StellaOps.Feedser.Source.Distro.Debian.Internal; +using StellaOps.Feedser.Storage.Mongo.Documents; + +namespace StellaOps.Feedser.Source.Distro.Debian.Tests; + +public sealed class DebianMapperTests +{ + [Fact] + public void Map_BuildsRangePrimitives_ForResolvedPackage() + { + var dto = new DebianAdvisoryDto( + AdvisoryId: "DSA-2024-123", + SourcePackage: "openssl", + Title: "Openssl security update", + Description: "Fixes multiple issues.", + CveIds: new[] { "CVE-2024-1000", "CVE-2024-1001" }, + Packages: new[] + { + new DebianPackageStateDto( + Package: "openssl", + Release: "bullseye", + Status: "resolved", + IntroducedVersion: "1:1.1.1n-0+deb11u2", + FixedVersion: "1:1.1.1n-0+deb11u5", + LastAffectedVersion: null, + Published: new DateTimeOffset(2024, 9, 1, 0, 0, 0, TimeSpan.Zero)), + new DebianPackageStateDto( + Package: "openssl", + Release: "bookworm", + Status: "open", + IntroducedVersion: null, + FixedVersion: null, + LastAffectedVersion: null, + Published: null) + }, + References: new[] + { + new DebianReferenceDto( + Url: "https://security-tracker.debian.org/tracker/DSA-2024-123", + Kind: "advisory", + Title: "Debian Security Advisory 2024-123"), + }); + + var document = new DocumentRecord( + Id: Guid.NewGuid(), + SourceName: DebianConnectorPlugin.SourceName, + Uri: "https://security-tracker.debian.org/tracker/DSA-2024-123", + FetchedAt: new DateTimeOffset(2024, 9, 1, 1, 0, 0, TimeSpan.Zero), + Sha256: "sha", + Status: "Fetched", + ContentType: "application/json", + Headers: null, + Metadata: null, + Etag: null, + LastModified: null, + GridFsId: null); + + Advisory advisory = DebianMapper.Map(dto, document, new DateTimeOffset(2024, 9, 1, 2, 0, 0, TimeSpan.Zero)); + + Assert.Equal("DSA-2024-123", advisory.AdvisoryKey); + Assert.Contains("CVE-2024-1000", advisory.Aliases); + Assert.Contains("CVE-2024-1001", advisory.Aliases); + + var resolvedPackage = Assert.Single(advisory.AffectedPackages, p => p.Platform == "bullseye"); + var range = Assert.Single(resolvedPackage.VersionRanges); + Assert.Equal("evr", range.RangeKind); + Assert.Equal("1:1.1.1n-0+deb11u2", range.IntroducedVersion); + Assert.Equal("1:1.1.1n-0+deb11u5", range.FixedVersion); + Assert.NotNull(range.Primitives); + var evr = range.Primitives!.Evr; + Assert.NotNull(evr); + Assert.NotNull(evr!.Introduced); + Assert.Equal(1, evr.Introduced!.Epoch); + Assert.Equal("1.1.1n", evr.Introduced.UpstreamVersion); + Assert.Equal("0+deb11u2", evr.Introduced.Revision); + Assert.NotNull(evr.Fixed); + Assert.Equal(1, evr.Fixed!.Epoch); + Assert.Equal("1.1.1n", evr.Fixed.UpstreamVersion); + Assert.Equal("0+deb11u5", evr.Fixed.Revision); + + var openPackage = Assert.Single(advisory.AffectedPackages, p => p.Platform == "bookworm"); + Assert.Empty(openPackage.VersionRanges); + } +} diff --git a/src/StellaOps.Feedser.Source.Distro.Debian.Tests/Source/Distro/Debian/Fixtures/debian-detail-dsa-2024-123.html b/src/StellaOps.Feedser.Source.Distro.Debian.Tests/Source/Distro/Debian/Fixtures/debian-detail-dsa-2024-123.html new file mode 100644 index 00000000..278540ab --- /dev/null +++ b/src/StellaOps.Feedser.Source.Distro.Debian.Tests/Source/Distro/Debian/Fixtures/debian-detail-dsa-2024-123.html @@ -0,0 +1,23 @@ + + + + DSA-2024-123 + + +

    DSA-2024-123

    + + + + + +
    NameDSA-2024-123
    Descriptionopenssl - security update
    SourceDebian
    ReferencesCVE-2024-1000, CVE-2024-1001
    +

    Vulnerable and fixed packages

    + + + + + + +
    Source PackageReleaseVersionStatus
    opensslbookworm1:1.1.1n-0+deb11u2vulnerable
    bookworm (security)1:1.1.1n-0+deb11u5fixed
    trixie3.0.8-2vulnerable
    trixie (security)3.0.12-1fixed
    + + diff --git a/src/StellaOps.Feedser.Source.Distro.Debian.Tests/Source/Distro/Debian/Fixtures/debian-detail-dsa-2024-124.html b/src/StellaOps.Feedser.Source.Distro.Debian.Tests/Source/Distro/Debian/Fixtures/debian-detail-dsa-2024-124.html new file mode 100644 index 00000000..301e9901 --- /dev/null +++ b/src/StellaOps.Feedser.Source.Distro.Debian.Tests/Source/Distro/Debian/Fixtures/debian-detail-dsa-2024-124.html @@ -0,0 +1,21 @@ + + + + DSA-2024-124 + + +

    DSA-2024-124

    + + + + + +
    NameDSA-2024-124
    Descriptionzlib - security update
    SourceDebian
    ReferencesCVE-2024-2000
    +

    Vulnerable and fixed packages

    + + + + +
    Source PackageReleaseVersionStatus
    zlibbookworm1:1.3.1-1vulnerable
    trixie1:1.3.1-2vulnerable
    + + diff --git a/src/StellaOps.Feedser.Source.Distro.Debian.Tests/Source/Distro/Debian/Fixtures/debian-list.txt b/src/StellaOps.Feedser.Source.Distro.Debian.Tests/Source/Distro/Debian/Fixtures/debian-list.txt new file mode 100644 index 00000000..6ee359bb --- /dev/null +++ b/src/StellaOps.Feedser.Source.Distro.Debian.Tests/Source/Distro/Debian/Fixtures/debian-list.txt @@ -0,0 +1,7 @@ +[12 Sep 2024] DSA-2024-123 openssl - security update + {CVE-2024-1000 CVE-2024-1001} + [bookworm] - openssl 1:1.1.1n-0+deb11u5 + [trixie] - openssl 3.0.12-1 +[10 Sep 2024] DSA-2024-124 zlib - security update + {CVE-2024-2000} + [bookworm] - zlib 1:1.3.2-1 diff --git a/src/StellaOps.Feedser.Source.Distro.Debian.Tests/StellaOps.Feedser.Source.Distro.Debian.Tests.csproj b/src/StellaOps.Feedser.Source.Distro.Debian.Tests/StellaOps.Feedser.Source.Distro.Debian.Tests.csproj new file mode 100644 index 00000000..d75a27bb --- /dev/null +++ b/src/StellaOps.Feedser.Source.Distro.Debian.Tests/StellaOps.Feedser.Source.Distro.Debian.Tests.csproj @@ -0,0 +1,13 @@ + + + net10.0 + enable + enable + + + + + + + + diff --git a/src/StellaOps.Feedser.Source.Distro.Debian/AssemblyInfo.cs b/src/StellaOps.Feedser.Source.Distro.Debian/AssemblyInfo.cs new file mode 100644 index 00000000..e2c83a72 --- /dev/null +++ b/src/StellaOps.Feedser.Source.Distro.Debian/AssemblyInfo.cs @@ -0,0 +1,3 @@ +using System.Runtime.CompilerServices; + +[assembly: InternalsVisibleTo("StellaOps.Feedser.Source.Distro.Debian.Tests")] diff --git a/src/StellaOps.Feedser.Source.Distro.Debian/Configuration/DebianOptions.cs b/src/StellaOps.Feedser.Source.Distro.Debian/Configuration/DebianOptions.cs new file mode 100644 index 00000000..c5c82e4a --- /dev/null +++ b/src/StellaOps.Feedser.Source.Distro.Debian/Configuration/DebianOptions.cs @@ -0,0 +1,87 @@ +using System; + +namespace StellaOps.Feedser.Source.Distro.Debian.Configuration; + +public sealed class DebianOptions +{ + public const string HttpClientName = "feedser.debian"; + + /// + /// Raw advisory list published by the Debian security tracker team. + /// Defaults to the Salsa Git raw endpoint to avoid HTML scraping. + /// + public Uri ListEndpoint { get; set; } = new("https://salsa.debian.org/security-tracker-team/security-tracker/-/raw/master/data/DSA/list"); + + /// + /// Base URI for advisory detail pages. Connector appends {AdvisoryId}. + /// + public Uri DetailBaseUri { get; set; } = new("https://security-tracker.debian.org/tracker/"); + + /// + /// Maximum advisories fetched per run to cap backfill effort. + /// + public int MaxAdvisoriesPerFetch { get; set; } = 40; + + /// + /// Initial history window pulled on first run. + /// + public TimeSpan InitialBackfill { get; set; } = TimeSpan.FromDays(30); + + /// + /// Resume overlap to accommodate late edits of existing advisories. + /// + public TimeSpan ResumeOverlap { get; set; } = TimeSpan.FromDays(2); + + /// + /// Request timeout used for list/detail fetches unless overridden via HTTP client. + /// + public TimeSpan FetchTimeout { get; set; } = TimeSpan.FromSeconds(45); + + /// + /// Optional pacing delay between detail fetches. + /// + public TimeSpan RequestDelay { get; set; } = TimeSpan.Zero; + + /// + /// Custom user-agent for Debian tracker courtesy. + /// + public string UserAgent { get; set; } = "StellaOps.Feedser.Debian/0.1 (+https://stella-ops.org)"; + + public void Validate() + { + if (ListEndpoint is null || !ListEndpoint.IsAbsoluteUri) + { + throw new InvalidOperationException("Debian list endpoint must be an absolute URI."); + } + + if (DetailBaseUri is null || !DetailBaseUri.IsAbsoluteUri) + { + throw new InvalidOperationException("Debian detail base URI must be an absolute URI."); + } + + if (MaxAdvisoriesPerFetch <= 0 || MaxAdvisoriesPerFetch > 200) + { + throw new InvalidOperationException("MaxAdvisoriesPerFetch must be between 1 and 200."); + } + + if (InitialBackfill < TimeSpan.Zero || InitialBackfill > TimeSpan.FromDays(365)) + { + throw new InvalidOperationException("InitialBackfill must be between 0 and 365 days."); + } + + if (ResumeOverlap < TimeSpan.Zero || ResumeOverlap > TimeSpan.FromDays(14)) + { + throw new InvalidOperationException("ResumeOverlap must be between 0 and 14 days."); + } + + if (FetchTimeout <= TimeSpan.Zero || FetchTimeout > TimeSpan.FromMinutes(5)) + { + throw new InvalidOperationException("FetchTimeout must be positive and less than five minutes."); + } + + if (RequestDelay < TimeSpan.Zero || RequestDelay > TimeSpan.FromSeconds(10)) + { + throw new InvalidOperationException("RequestDelay must be between 0 and 10 seconds."); + } + } +} diff --git a/src/StellaOps.Feedser.Source.Distro.Debian/DebianConnector.cs b/src/StellaOps.Feedser.Source.Distro.Debian/DebianConnector.cs new file mode 100644 index 00000000..ddd1c1de --- /dev/null +++ b/src/StellaOps.Feedser.Source.Distro.Debian/DebianConnector.cs @@ -0,0 +1,637 @@ +using System; +using System.Collections.Generic; +using System.Globalization; +using System.Linq; +using System.Net; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Options; +using MongoDB.Bson; +using MongoDB.Bson.IO; +using StellaOps.Feedser.Models; +using StellaOps.Feedser.Source.Common; +using StellaOps.Feedser.Source.Common.Fetch; +using StellaOps.Feedser.Source.Distro.Debian.Configuration; +using StellaOps.Feedser.Source.Distro.Debian.Internal; +using StellaOps.Feedser.Storage.Mongo; +using StellaOps.Feedser.Storage.Mongo.Advisories; +using StellaOps.Feedser.Storage.Mongo.Documents; +using StellaOps.Feedser.Storage.Mongo.Dtos; +using StellaOps.Plugin; + +namespace StellaOps.Feedser.Source.Distro.Debian; + +public sealed class DebianConnector : IFeedConnector +{ + private const string SchemaVersion = "debian.v1"; + + private readonly SourceFetchService _fetchService; + private readonly RawDocumentStorage _rawDocumentStorage; + private readonly IDocumentStore _documentStore; + private readonly IDtoStore _dtoStore; + private readonly IAdvisoryStore _advisoryStore; + private readonly ISourceStateRepository _stateRepository; + private readonly DebianOptions _options; + private readonly TimeProvider _timeProvider; + private readonly ILogger _logger; + + private static readonly Action LogMapped = + LoggerMessage.Define( + LogLevel.Information, + new EventId(1, "DebianMapped"), + "Debian advisory {AdvisoryId} mapped with {AffectedCount} packages"); + + public DebianConnector( + SourceFetchService fetchService, + RawDocumentStorage rawDocumentStorage, + IDocumentStore documentStore, + IDtoStore dtoStore, + IAdvisoryStore advisoryStore, + ISourceStateRepository stateRepository, + IOptions options, + TimeProvider? timeProvider, + ILogger logger) + { + _fetchService = fetchService ?? throw new ArgumentNullException(nameof(fetchService)); + _rawDocumentStorage = rawDocumentStorage ?? throw new ArgumentNullException(nameof(rawDocumentStorage)); + _documentStore = documentStore ?? throw new ArgumentNullException(nameof(documentStore)); + _dtoStore = dtoStore ?? throw new ArgumentNullException(nameof(dtoStore)); + _advisoryStore = advisoryStore ?? throw new ArgumentNullException(nameof(advisoryStore)); + _stateRepository = stateRepository ?? throw new ArgumentNullException(nameof(stateRepository)); + _options = (options ?? throw new ArgumentNullException(nameof(options))).Value ?? throw new ArgumentNullException(nameof(options)); + _options.Validate(); + _timeProvider = timeProvider ?? TimeProvider.System; + _logger = logger ?? throw new ArgumentNullException(nameof(logger)); + } + + public string SourceName => DebianConnectorPlugin.SourceName; + + public async Task FetchAsync(IServiceProvider services, CancellationToken cancellationToken) + { + ArgumentNullException.ThrowIfNull(services); + + var cursor = await GetCursorAsync(cancellationToken).ConfigureAwait(false); + var now = _timeProvider.GetUtcNow(); + + var pendingDocuments = new HashSet(cursor.PendingDocuments); + var pendingMappings = new HashSet(cursor.PendingMappings); + var fetchCache = new Dictionary(cursor.FetchCache, StringComparer.OrdinalIgnoreCase); + var touchedResources = new HashSet(StringComparer.OrdinalIgnoreCase); + + var listUri = _options.ListEndpoint; + var listKey = listUri.ToString(); + touchedResources.Add(listKey); + + var existingList = await _documentStore.FindBySourceAndUriAsync(SourceName, listKey, cancellationToken).ConfigureAwait(false); + cursor.TryGetCache(listKey, out var cachedListEntry); + + var listRequest = new SourceFetchRequest(DebianOptions.HttpClientName, SourceName, listUri) + { + Metadata = new Dictionary(StringComparer.Ordinal) + { + ["type"] = "index" + }, + AcceptHeaders = new[] { "text/plain", "text/plain; charset=utf-8" }, + TimeoutOverride = _options.FetchTimeout, + ETag = existingList?.Etag ?? cachedListEntry?.ETag, + LastModified = existingList?.LastModified ?? cachedListEntry?.LastModified, + }; + + SourceFetchResult listResult; + try + { + listResult = await _fetchService.FetchAsync(listRequest, cancellationToken).ConfigureAwait(false); + } + catch (Exception ex) + { + _logger.LogError(ex, "Debian list fetch failed"); + await _stateRepository.MarkFailureAsync(SourceName, now, TimeSpan.FromMinutes(5), ex.Message, cancellationToken).ConfigureAwait(false); + throw; + } + + var lastPublished = cursor.LastPublished ?? (now - _options.InitialBackfill); + var processedIds = new HashSet(cursor.ProcessedAdvisoryIds, StringComparer.OrdinalIgnoreCase); + var newProcessedIds = new HashSet(StringComparer.OrdinalIgnoreCase); + var maxPublished = cursor.LastPublished ?? DateTimeOffset.MinValue; + var processedUpdated = false; + + if (listResult.IsNotModified) + { + if (existingList is not null) + { + fetchCache[listKey] = DebianFetchCacheEntry.FromDocument(existingList); + } + } + else if (listResult.IsSuccess && listResult.Document is not null) + { + fetchCache[listKey] = DebianFetchCacheEntry.FromDocument(listResult.Document); + + if (!listResult.Document.GridFsId.HasValue) + { + _logger.LogWarning("Debian list document {DocumentId} missing GridFS payload", listResult.Document.Id); + } + else + { + byte[] bytes; + try + { + bytes = await _rawDocumentStorage.DownloadAsync(listResult.Document.GridFsId.Value, cancellationToken).ConfigureAwait(false); + } + catch (Exception ex) + { + _logger.LogError(ex, "Failed to download Debian list document {DocumentId}", listResult.Document.Id); + throw; + } + + var text = System.Text.Encoding.UTF8.GetString(bytes); + var entries = DebianListParser.Parse(text); + if (entries.Count > 0) + { + var windowStart = (cursor.LastPublished ?? (now - _options.InitialBackfill)) - _options.ResumeOverlap; + if (windowStart < DateTimeOffset.UnixEpoch) + { + windowStart = DateTimeOffset.UnixEpoch; + } + + ProvenanceDiagnostics.ReportResumeWindow(SourceName, windowStart, _logger); + + var candidates = entries + .Where(entry => entry.Published >= windowStart) + .OrderBy(entry => entry.Published) + .ThenBy(entry => entry.AdvisoryId, StringComparer.OrdinalIgnoreCase) + .ToList(); + + if (candidates.Count == 0) + { + candidates = entries + .OrderByDescending(entry => entry.Published) + .ThenBy(entry => entry.AdvisoryId, StringComparer.OrdinalIgnoreCase) + .Take(_options.MaxAdvisoriesPerFetch) + .OrderBy(entry => entry.Published) + .ThenBy(entry => entry.AdvisoryId, StringComparer.OrdinalIgnoreCase) + .ToList(); + } + else if (candidates.Count > _options.MaxAdvisoriesPerFetch) + { + candidates = candidates + .OrderByDescending(entry => entry.Published) + .ThenBy(entry => entry.AdvisoryId, StringComparer.OrdinalIgnoreCase) + .Take(_options.MaxAdvisoriesPerFetch) + .OrderBy(entry => entry.Published) + .ThenBy(entry => entry.AdvisoryId, StringComparer.OrdinalIgnoreCase) + .ToList(); + } + + foreach (var entry in candidates) + { + cancellationToken.ThrowIfCancellationRequested(); + + var detailUri = new Uri(_options.DetailBaseUri, entry.AdvisoryId); + var cacheKey = detailUri.ToString(); + touchedResources.Add(cacheKey); + + cursor.TryGetCache(cacheKey, out var cachedDetail); + if (!fetchCache.TryGetValue(cacheKey, out var cachedInRun)) + { + cachedInRun = cachedDetail; + } + + var metadata = BuildDetailMetadata(entry); + var existingDetail = await _documentStore.FindBySourceAndUriAsync(SourceName, cacheKey, cancellationToken).ConfigureAwait(false); + + var request = new SourceFetchRequest(DebianOptions.HttpClientName, SourceName, detailUri) + { + Metadata = metadata, + AcceptHeaders = new[] { "text/html", "application/xhtml+xml" }, + TimeoutOverride = _options.FetchTimeout, + ETag = existingDetail?.Etag ?? cachedInRun?.ETag, + LastModified = existingDetail?.LastModified ?? cachedInRun?.LastModified, + }; + + SourceFetchResult result; + try + { + result = await _fetchService.FetchAsync(request, cancellationToken).ConfigureAwait(false); + } + catch (Exception ex) + { + _logger.LogError(ex, "Failed to fetch Debian advisory {AdvisoryId}", entry.AdvisoryId); + await _stateRepository.MarkFailureAsync(SourceName, now, TimeSpan.FromMinutes(5), ex.Message, cancellationToken).ConfigureAwait(false); + throw; + } + + if (result.IsNotModified) + { + if (existingDetail is not null) + { + fetchCache[cacheKey] = DebianFetchCacheEntry.FromDocument(existingDetail); + if (string.Equals(existingDetail.Status, DocumentStatuses.Mapped, StringComparison.Ordinal)) + { + pendingDocuments.Remove(existingDetail.Id); + pendingMappings.Remove(existingDetail.Id); + } + } + + continue; + } + + if (!result.IsSuccess || result.Document is null) + { + continue; + } + + fetchCache[cacheKey] = DebianFetchCacheEntry.FromDocument(result.Document); + pendingDocuments.Add(result.Document.Id); + pendingMappings.Remove(result.Document.Id); + + if (_options.RequestDelay > TimeSpan.Zero) + { + try + { + await Task.Delay(_options.RequestDelay, cancellationToken).ConfigureAwait(false); + } + catch (TaskCanceledException) + { + break; + } + } + + if (entry.Published > maxPublished) + { + maxPublished = entry.Published; + newProcessedIds.Clear(); + processedUpdated = true; + } + + if (entry.Published == maxPublished) + { + newProcessedIds.Add(entry.AdvisoryId); + processedUpdated = true; + } + } + } + } + } + + if (fetchCache.Count > 0 && touchedResources.Count > 0) + { + var stale = fetchCache.Keys.Where(key => !touchedResources.Contains(key)).ToArray(); + foreach (var key in stale) + { + fetchCache.Remove(key); + } + } + + if (!processedUpdated && cursor.LastPublished.HasValue) + { + maxPublished = cursor.LastPublished.Value; + newProcessedIds = new HashSet(cursor.ProcessedAdvisoryIds, StringComparer.OrdinalIgnoreCase); + } + + var updatedCursor = cursor + .WithPendingDocuments(pendingDocuments) + .WithPendingMappings(pendingMappings) + .WithFetchCache(fetchCache); + + if (processedUpdated && maxPublished > DateTimeOffset.MinValue) + { + updatedCursor = updatedCursor.WithProcessed(maxPublished, newProcessedIds); + } + + await UpdateCursorAsync(updatedCursor, cancellationToken).ConfigureAwait(false); + } + + public async Task ParseAsync(IServiceProvider services, CancellationToken cancellationToken) + { + ArgumentNullException.ThrowIfNull(services); + + var cursor = await GetCursorAsync(cancellationToken).ConfigureAwait(false); + if (cursor.PendingDocuments.Count == 0) + { + return; + } + + var remaining = cursor.PendingDocuments.ToList(); + var pendingMappings = cursor.PendingMappings.ToList(); + + foreach (var documentId in cursor.PendingDocuments) + { + cancellationToken.ThrowIfCancellationRequested(); + + var document = await _documentStore.FindAsync(documentId, cancellationToken).ConfigureAwait(false); + if (document is null) + { + remaining.Remove(documentId); + continue; + } + + if (!document.GridFsId.HasValue) + { + _logger.LogWarning("Debian document {DocumentId} missing GridFS payload", document.Id); + await _documentStore.UpdateStatusAsync(document.Id, DocumentStatuses.Failed, cancellationToken).ConfigureAwait(false); + remaining.Remove(documentId); + continue; + } + + var metadata = ExtractMetadata(document); + if (metadata is null) + { + _logger.LogWarning("Debian document {DocumentId} missing required metadata", document.Id); + await _documentStore.UpdateStatusAsync(document.Id, DocumentStatuses.Failed, cancellationToken).ConfigureAwait(false); + remaining.Remove(documentId); + continue; + } + + byte[] bytes; + try + { + bytes = await _rawDocumentStorage.DownloadAsync(document.GridFsId.Value, cancellationToken).ConfigureAwait(false); + } + catch (Exception ex) + { + _logger.LogError(ex, "Failed to download Debian document {DocumentId}", document.Id); + throw; + } + + var html = System.Text.Encoding.UTF8.GetString(bytes); + DebianAdvisoryDto dto; + try + { + dto = DebianHtmlParser.Parse(html, metadata); + } + catch (Exception ex) + { + _logger.LogWarning(ex, "Failed to parse Debian advisory {AdvisoryId}", metadata.AdvisoryId); + await _documentStore.UpdateStatusAsync(document.Id, DocumentStatuses.Failed, cancellationToken).ConfigureAwait(false); + remaining.Remove(document.Id); + continue; + } + + var payload = ToBson(dto); + var dtoRecord = new DtoRecord(Guid.NewGuid(), document.Id, SourceName, SchemaVersion, payload, _timeProvider.GetUtcNow()); + await _dtoStore.UpsertAsync(dtoRecord, cancellationToken).ConfigureAwait(false); + await _documentStore.UpdateStatusAsync(document.Id, DocumentStatuses.PendingMap, cancellationToken).ConfigureAwait(false); + + remaining.Remove(document.Id); + if (!pendingMappings.Contains(document.Id)) + { + pendingMappings.Add(document.Id); + } + } + + var updatedCursor = cursor + .WithPendingDocuments(remaining) + .WithPendingMappings(pendingMappings); + + await UpdateCursorAsync(updatedCursor, cancellationToken).ConfigureAwait(false); + } + + public async Task MapAsync(IServiceProvider services, CancellationToken cancellationToken) + { + ArgumentNullException.ThrowIfNull(services); + + var cursor = await GetCursorAsync(cancellationToken).ConfigureAwait(false); + if (cursor.PendingMappings.Count == 0) + { + return; + } + + var pendingMappings = cursor.PendingMappings.ToList(); + + foreach (var documentId in cursor.PendingMappings) + { + cancellationToken.ThrowIfCancellationRequested(); + + var dtoRecord = await _dtoStore.FindByDocumentIdAsync(documentId, cancellationToken).ConfigureAwait(false); + var document = await _documentStore.FindAsync(documentId, cancellationToken).ConfigureAwait(false); + if (dtoRecord is null || document is null) + { + pendingMappings.Remove(documentId); + continue; + } + + DebianAdvisoryDto dto; + try + { + dto = FromBson(dtoRecord.Payload); + } + catch (Exception ex) + { + _logger.LogError(ex, "Failed to deserialize Debian DTO for document {DocumentId}", documentId); + await _documentStore.UpdateStatusAsync(documentId, DocumentStatuses.Failed, cancellationToken).ConfigureAwait(false); + pendingMappings.Remove(documentId); + continue; + } + + var advisory = DebianMapper.Map(dto, document, _timeProvider.GetUtcNow()); + await _advisoryStore.UpsertAsync(advisory, cancellationToken).ConfigureAwait(false); + await _documentStore.UpdateStatusAsync(documentId, DocumentStatuses.Mapped, cancellationToken).ConfigureAwait(false); + pendingMappings.Remove(documentId); + LogMapped(_logger, dto.AdvisoryId, advisory.AffectedPackages.Length, null); + } + + var updatedCursor = cursor.WithPendingMappings(pendingMappings); + await UpdateCursorAsync(updatedCursor, cancellationToken).ConfigureAwait(false); + } + + private async Task GetCursorAsync(CancellationToken cancellationToken) + { + var state = await _stateRepository.TryGetAsync(SourceName, cancellationToken).ConfigureAwait(false); + return state is null ? DebianCursor.Empty : DebianCursor.FromBson(state.Cursor); + } + + private async Task UpdateCursorAsync(DebianCursor cursor, CancellationToken cancellationToken) + { + var document = cursor.ToBsonDocument(); + await _stateRepository.UpdateCursorAsync(SourceName, document, _timeProvider.GetUtcNow(), cancellationToken).ConfigureAwait(false); + } + + private static Dictionary BuildDetailMetadata(DebianListEntry entry) + { + var metadata = new Dictionary(StringComparer.Ordinal) + { + ["debian.id"] = entry.AdvisoryId, + ["debian.published"] = entry.Published.ToString("O", CultureInfo.InvariantCulture), + ["debian.title"] = entry.Title, + ["debian.package"] = entry.SourcePackage + }; + + if (entry.CveIds.Count > 0) + { + metadata["debian.cves"] = string.Join(' ', entry.CveIds); + } + + return metadata; + } + + private static DebianDetailMetadata? ExtractMetadata(DocumentRecord document) + { + if (document.Metadata is null) + { + return null; + } + + if (!document.Metadata.TryGetValue("debian.id", out var id) || string.IsNullOrWhiteSpace(id)) + { + return null; + } + + if (!document.Metadata.TryGetValue("debian.published", out var publishedRaw) + || !DateTimeOffset.TryParse(publishedRaw, CultureInfo.InvariantCulture, DateTimeStyles.AssumeUniversal, out var published)) + { + published = document.FetchedAt; + } + + var title = document.Metadata.TryGetValue("debian.title", out var t) ? t : id; + var package = document.Metadata.TryGetValue("debian.package", out var pkg) && !string.IsNullOrWhiteSpace(pkg) + ? pkg + : id; + + IReadOnlyList cveList = Array.Empty(); + if (document.Metadata.TryGetValue("debian.cves", out var cvesRaw) && !string.IsNullOrWhiteSpace(cvesRaw)) + { + cveList = cvesRaw + .Split(' ', StringSplitOptions.TrimEntries | StringSplitOptions.RemoveEmptyEntries) + .Where(static s => !string.IsNullOrWhiteSpace(s)) + .Select(static s => s!) + .Distinct(StringComparer.OrdinalIgnoreCase) + .ToArray(); + } + + return new DebianDetailMetadata( + id.Trim(), + new Uri(document.Uri, UriKind.Absolute), + published.ToUniversalTime(), + title, + package, + cveList); + } + + private static BsonDocument ToBson(DebianAdvisoryDto dto) + { + var packages = new BsonArray(); + foreach (var package in dto.Packages) + { + var packageDoc = new BsonDocument + { + ["package"] = package.Package, + ["release"] = package.Release, + ["status"] = package.Status, + }; + + if (!string.IsNullOrWhiteSpace(package.IntroducedVersion)) + { + packageDoc["introduced"] = package.IntroducedVersion; + } + + if (!string.IsNullOrWhiteSpace(package.FixedVersion)) + { + packageDoc["fixed"] = package.FixedVersion; + } + + if (!string.IsNullOrWhiteSpace(package.LastAffectedVersion)) + { + packageDoc["last"] = package.LastAffectedVersion; + } + + if (package.Published.HasValue) + { + packageDoc["published"] = package.Published.Value.UtcDateTime; + } + + packages.Add(packageDoc); + } + + var references = new BsonArray(dto.References.Select(reference => + { + var doc = new BsonDocument + { + ["url"] = reference.Url + }; + + if (!string.IsNullOrWhiteSpace(reference.Kind)) + { + doc["kind"] = reference.Kind; + } + + if (!string.IsNullOrWhiteSpace(reference.Title)) + { + doc["title"] = reference.Title; + } + + return doc; + })); + + return new BsonDocument + { + ["advisoryId"] = dto.AdvisoryId, + ["sourcePackage"] = dto.SourcePackage, + ["title"] = dto.Title, + ["description"] = dto.Description ?? string.Empty, + ["cves"] = new BsonArray(dto.CveIds), + ["packages"] = packages, + ["references"] = references, + }; + } + + private static DebianAdvisoryDto FromBson(BsonDocument document) + { + var advisoryId = document.GetValue("advisoryId", "").AsString; + var sourcePackage = document.GetValue("sourcePackage", advisoryId).AsString; + var title = document.GetValue("title", advisoryId).AsString; + var description = document.TryGetValue("description", out var desc) ? desc.AsString : null; + + var cves = document.TryGetValue("cves", out var cveArray) && cveArray is BsonArray cvesBson + ? cvesBson.OfType() + .Select(static value => value.ToString()) + .Where(static s => !string.IsNullOrWhiteSpace(s)) + .Select(static s => s!) + .ToArray() + : Array.Empty(); + + var packages = new List(); + if (document.TryGetValue("packages", out var packageArray) && packageArray is BsonArray packagesBson) + { + foreach (var element in packagesBson.OfType()) + { + packages.Add(new DebianPackageStateDto( + element.GetValue("package", sourcePackage).AsString, + element.GetValue("release", string.Empty).AsString, + element.GetValue("status", "unknown").AsString, + element.TryGetValue("introduced", out var introducedValue) ? introducedValue.AsString : null, + element.TryGetValue("fixed", out var fixedValue) ? fixedValue.AsString : null, + element.TryGetValue("last", out var lastValue) ? lastValue.AsString : null, + element.TryGetValue("published", out var publishedValue) + ? publishedValue.BsonType switch + { + BsonType.DateTime => DateTime.SpecifyKind(publishedValue.ToUniversalTime(), DateTimeKind.Utc), + BsonType.String when DateTimeOffset.TryParse(publishedValue.AsString, out var parsed) => parsed.ToUniversalTime(), + _ => (DateTimeOffset?)null, + } + : null)); + } + } + + var references = new List(); + if (document.TryGetValue("references", out var referenceArray) && referenceArray is BsonArray refBson) + { + foreach (var element in refBson.OfType()) + { + references.Add(new DebianReferenceDto( + element.GetValue("url", "").AsString, + element.TryGetValue("kind", out var kind) ? kind.AsString : null, + element.TryGetValue("title", out var titleValue) ? titleValue.AsString : null)); + } + } + + return new DebianAdvisoryDto( + advisoryId, + sourcePackage, + title, + description, + cves, + packages, + references); + } +} diff --git a/src/StellaOps.Feedser.Source.Distro.Debian/DebianConnectorPlugin.cs b/src/StellaOps.Feedser.Source.Distro.Debian/DebianConnectorPlugin.cs new file mode 100644 index 00000000..5f4aced7 --- /dev/null +++ b/src/StellaOps.Feedser.Source.Distro.Debian/DebianConnectorPlugin.cs @@ -0,0 +1,22 @@ +using System; +using System.Threading; +using System; +using Microsoft.Extensions.DependencyInjection; +using StellaOps.Plugin; + +namespace StellaOps.Feedser.Source.Distro.Debian; + +public sealed class DebianConnectorPlugin : IConnectorPlugin +{ + public const string SourceName = "distro-debian"; + + public string Name => SourceName; + + public bool IsAvailable(IServiceProvider services) => services is not null; + + public IFeedConnector Create(IServiceProvider services) + { + ArgumentNullException.ThrowIfNull(services); + return ActivatorUtilities.CreateInstance(services); + } +} diff --git a/src/StellaOps.Feedser.Source.Distro.Debian/DebianDependencyInjectionRoutine.cs b/src/StellaOps.Feedser.Source.Distro.Debian/DebianDependencyInjectionRoutine.cs new file mode 100644 index 00000000..562ded3d --- /dev/null +++ b/src/StellaOps.Feedser.Source.Distro.Debian/DebianDependencyInjectionRoutine.cs @@ -0,0 +1,53 @@ +using System; +using Microsoft.Extensions.Configuration; +using Microsoft.Extensions.DependencyInjection; +using StellaOps.DependencyInjection; +using StellaOps.Feedser.Core.Jobs; +using StellaOps.Feedser.Source.Distro.Debian.Configuration; + +namespace StellaOps.Feedser.Source.Distro.Debian; + +public sealed class DebianDependencyInjectionRoutine : IDependencyInjectionRoutine +{ + private const string ConfigurationSection = "feedser:sources:debian"; + private const string FetchSchedule = "*/30 * * * *"; + private const string ParseSchedule = "7,37 * * * *"; + private const string MapSchedule = "12,42 * * * *"; + + private static readonly TimeSpan FetchTimeout = TimeSpan.FromMinutes(6); + private static readonly TimeSpan ParseTimeout = TimeSpan.FromMinutes(10); + private static readonly TimeSpan MapTimeout = TimeSpan.FromMinutes(10); + private static readonly TimeSpan LeaseDuration = TimeSpan.FromMinutes(5); + + public IServiceCollection Register(IServiceCollection services, IConfiguration configuration) + { + ArgumentNullException.ThrowIfNull(services); + ArgumentNullException.ThrowIfNull(configuration); + + services.AddDebianConnector(options => + { + configuration.GetSection(ConfigurationSection).Bind(options); + options.Validate(); + }); + + var scheduler = new JobSchedulerBuilder(services); + scheduler + .AddJob( + DebianJobKinds.Fetch, + cronExpression: FetchSchedule, + timeout: FetchTimeout, + leaseDuration: LeaseDuration) + .AddJob( + DebianJobKinds.Parse, + cronExpression: ParseSchedule, + timeout: ParseTimeout, + leaseDuration: LeaseDuration) + .AddJob( + DebianJobKinds.Map, + cronExpression: MapSchedule, + timeout: MapTimeout, + leaseDuration: LeaseDuration); + + return services; + } +} diff --git a/src/StellaOps.Feedser.Source.Distro.Debian/DebianServiceCollectionExtensions.cs b/src/StellaOps.Feedser.Source.Distro.Debian/DebianServiceCollectionExtensions.cs new file mode 100644 index 00000000..5df031df --- /dev/null +++ b/src/StellaOps.Feedser.Source.Distro.Debian/DebianServiceCollectionExtensions.cs @@ -0,0 +1,37 @@ +using System; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Options; +using StellaOps.Feedser.Source.Common.Http; +using StellaOps.Feedser.Source.Distro.Debian.Configuration; + +namespace StellaOps.Feedser.Source.Distro.Debian; + +public static class DebianServiceCollectionExtensions +{ + public static IServiceCollection AddDebianConnector(this IServiceCollection services, Action configure) + { + ArgumentNullException.ThrowIfNull(services); + ArgumentNullException.ThrowIfNull(configure); + + services.AddOptions() + .Configure(configure) + .PostConfigure(static options => options.Validate()); + + services.AddSourceHttpClient(DebianOptions.HttpClientName, (sp, httpOptions) => + { + var options = sp.GetRequiredService>().Value; + httpOptions.BaseAddress = options.DetailBaseUri.GetLeftPart(UriPartial.Authority) is { Length: > 0 } authority + ? new Uri(authority, UriKind.Absolute) + : new Uri("https://security-tracker.debian.org/", UriKind.Absolute); + httpOptions.Timeout = options.FetchTimeout; + httpOptions.UserAgent = options.UserAgent; + httpOptions.AllowedHosts.Clear(); + httpOptions.AllowedHosts.Add(options.DetailBaseUri.Host); + httpOptions.AllowedHosts.Add(options.ListEndpoint.Host); + httpOptions.DefaultRequestHeaders["Accept"] = "text/html,application/xhtml+xml,text/plain;q=0.9,application/json;q=0.8"; + }); + + services.AddTransient(); + return services; + } +} diff --git a/src/StellaOps.Feedser.Source.Distro.Debian/Internal/DebianAdvisoryDto.cs b/src/StellaOps.Feedser.Source.Distro.Debian/Internal/DebianAdvisoryDto.cs new file mode 100644 index 00000000..b1a88845 --- /dev/null +++ b/src/StellaOps.Feedser.Source.Distro.Debian/Internal/DebianAdvisoryDto.cs @@ -0,0 +1,27 @@ +using System; +using System.Collections.Generic; + +namespace StellaOps.Feedser.Source.Distro.Debian.Internal; + +internal sealed record DebianAdvisoryDto( + string AdvisoryId, + string SourcePackage, + string? Title, + string? Description, + IReadOnlyList CveIds, + IReadOnlyList Packages, + IReadOnlyList References); + +internal sealed record DebianPackageStateDto( + string Package, + string Release, + string Status, + string? IntroducedVersion, + string? FixedVersion, + string? LastAffectedVersion, + DateTimeOffset? Published); + +internal sealed record DebianReferenceDto( + string Url, + string? Kind, + string? Title); diff --git a/src/StellaOps.Feedser.Source.Distro.Debian/Internal/DebianCursor.cs b/src/StellaOps.Feedser.Source.Distro.Debian/Internal/DebianCursor.cs new file mode 100644 index 00000000..1b0d6bff --- /dev/null +++ b/src/StellaOps.Feedser.Source.Distro.Debian/Internal/DebianCursor.cs @@ -0,0 +1,177 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using MongoDB.Bson; + +namespace StellaOps.Feedser.Source.Distro.Debian.Internal; + +internal sealed record DebianCursor( + DateTimeOffset? LastPublished, + IReadOnlyCollection ProcessedAdvisoryIds, + IReadOnlyCollection PendingDocuments, + IReadOnlyCollection PendingMappings, + IReadOnlyDictionary FetchCache) +{ + private static readonly IReadOnlyCollection EmptyIds = Array.Empty(); + private static readonly IReadOnlyCollection EmptyGuidList = Array.Empty(); + private static readonly IReadOnlyDictionary EmptyCache = + new Dictionary(StringComparer.OrdinalIgnoreCase); + + public static DebianCursor Empty { get; } = new(null, EmptyIds, EmptyGuidList, EmptyGuidList, EmptyCache); + + public static DebianCursor FromBson(BsonDocument? document) + { + if (document is null || document.ElementCount == 0) + { + return Empty; + } + + DateTimeOffset? lastPublished = null; + if (document.TryGetValue("lastPublished", out var lastValue)) + { + lastPublished = lastValue.BsonType switch + { + BsonType.String when DateTimeOffset.TryParse(lastValue.AsString, out var parsed) => parsed.ToUniversalTime(), + BsonType.DateTime => DateTime.SpecifyKind(lastValue.ToUniversalTime(), DateTimeKind.Utc), + _ => null, + }; + } + + var processed = ReadStringArray(document, "processedIds"); + var pendingDocuments = ReadGuidArray(document, "pendingDocuments"); + var pendingMappings = ReadGuidArray(document, "pendingMappings"); + var cache = ReadCache(document); + + return new DebianCursor(lastPublished, processed, pendingDocuments, pendingMappings, cache); + } + + public BsonDocument ToBsonDocument() + { + var document = new BsonDocument + { + ["pendingDocuments"] = new BsonArray(PendingDocuments.Select(static id => id.ToString())), + ["pendingMappings"] = new BsonArray(PendingMappings.Select(static id => id.ToString())), + }; + + if (LastPublished.HasValue) + { + document["lastPublished"] = LastPublished.Value.UtcDateTime; + } + + if (ProcessedAdvisoryIds.Count > 0) + { + document["processedIds"] = new BsonArray(ProcessedAdvisoryIds); + } + + if (FetchCache.Count > 0) + { + var cacheDoc = new BsonDocument(); + foreach (var (key, entry) in FetchCache) + { + cacheDoc[key] = entry.ToBsonDocument(); + } + + document["fetchCache"] = cacheDoc; + } + + return document; + } + + public DebianCursor WithPendingDocuments(IEnumerable ids) + => this with { PendingDocuments = ids?.Distinct().ToArray() ?? EmptyGuidList }; + + public DebianCursor WithPendingMappings(IEnumerable ids) + => this with { PendingMappings = ids?.Distinct().ToArray() ?? EmptyGuidList }; + + public DebianCursor WithProcessed(DateTimeOffset published, IEnumerable ids) + => this with + { + LastPublished = published.ToUniversalTime(), + ProcessedAdvisoryIds = ids?.Where(static id => !string.IsNullOrWhiteSpace(id)) + .Select(static id => id.Trim()) + .Distinct(StringComparer.OrdinalIgnoreCase) + .ToArray() ?? EmptyIds + }; + + public DebianCursor WithFetchCache(IDictionary? cache) + { + if (cache is null || cache.Count == 0) + { + return this with { FetchCache = EmptyCache }; + } + + return this with { FetchCache = new Dictionary(cache, StringComparer.OrdinalIgnoreCase) }; + } + + public bool TryGetCache(string key, out DebianFetchCacheEntry entry) + { + if (FetchCache.Count == 0) + { + entry = DebianFetchCacheEntry.Empty; + return false; + } + + return FetchCache.TryGetValue(key, out entry!); + } + + private static IReadOnlyCollection ReadStringArray(BsonDocument document, string field) + { + if (!document.TryGetValue(field, out var value) || value is not BsonArray array) + { + return EmptyIds; + } + + var list = new List(array.Count); + foreach (var element in array) + { + if (element.BsonType == BsonType.String) + { + var str = element.AsString.Trim(); + if (!string.IsNullOrEmpty(str)) + { + list.Add(str); + } + } + } + + return list; + } + + private static IReadOnlyCollection ReadGuidArray(BsonDocument document, string field) + { + if (!document.TryGetValue(field, out var value) || value is not BsonArray array) + { + return EmptyGuidList; + } + + var list = new List(array.Count); + foreach (var element in array) + { + if (Guid.TryParse(element.ToString(), out var guid)) + { + list.Add(guid); + } + } + + return list; + } + + private static IReadOnlyDictionary ReadCache(BsonDocument document) + { + if (!document.TryGetValue("fetchCache", out var value) || value is not BsonDocument cacheDocument || cacheDocument.ElementCount == 0) + { + return EmptyCache; + } + + var cache = new Dictionary(StringComparer.OrdinalIgnoreCase); + foreach (var element in cacheDocument.Elements) + { + if (element.Value is BsonDocument entry) + { + cache[element.Name] = DebianFetchCacheEntry.FromBson(entry); + } + } + + return cache; + } +} diff --git a/src/StellaOps.Feedser.Source.Distro.Debian/Internal/DebianDetailMetadata.cs b/src/StellaOps.Feedser.Source.Distro.Debian/Internal/DebianDetailMetadata.cs new file mode 100644 index 00000000..56e95339 --- /dev/null +++ b/src/StellaOps.Feedser.Source.Distro.Debian/Internal/DebianDetailMetadata.cs @@ -0,0 +1,12 @@ +using System; +using System.Collections.Generic; + +namespace StellaOps.Feedser.Source.Distro.Debian.Internal; + +internal sealed record DebianDetailMetadata( + string AdvisoryId, + Uri DetailUri, + DateTimeOffset Published, + string Title, + string SourcePackage, + IReadOnlyList CveIds); diff --git a/src/StellaOps.Feedser.Source.Distro.Debian/Internal/DebianFetchCacheEntry.cs b/src/StellaOps.Feedser.Source.Distro.Debian/Internal/DebianFetchCacheEntry.cs new file mode 100644 index 00000000..3be35b6b --- /dev/null +++ b/src/StellaOps.Feedser.Source.Distro.Debian/Internal/DebianFetchCacheEntry.cs @@ -0,0 +1,76 @@ +using System; +using MongoDB.Bson; + +namespace StellaOps.Feedser.Source.Distro.Debian.Internal; + +internal sealed record DebianFetchCacheEntry(string? ETag, DateTimeOffset? LastModified) +{ + public static DebianFetchCacheEntry Empty { get; } = new(null, null); + + public static DebianFetchCacheEntry FromDocument(StellaOps.Feedser.Storage.Mongo.Documents.DocumentRecord document) + => new(document.Etag, document.LastModified); + + public static DebianFetchCacheEntry FromBson(BsonDocument document) + { + if (document is null || document.ElementCount == 0) + { + return Empty; + } + + string? etag = null; + DateTimeOffset? lastModified = null; + + if (document.TryGetValue("etag", out var etagValue) && etagValue.BsonType == BsonType.String) + { + etag = etagValue.AsString; + } + + if (document.TryGetValue("lastModified", out var modifiedValue)) + { + lastModified = modifiedValue.BsonType switch + { + BsonType.String when DateTimeOffset.TryParse(modifiedValue.AsString, out var parsed) => parsed.ToUniversalTime(), + BsonType.DateTime => DateTime.SpecifyKind(modifiedValue.ToUniversalTime(), DateTimeKind.Utc), + _ => null, + }; + } + + return new DebianFetchCacheEntry(etag, lastModified); + } + + public BsonDocument ToBsonDocument() + { + var document = new BsonDocument(); + if (!string.IsNullOrWhiteSpace(ETag)) + { + document["etag"] = ETag; + } + + if (LastModified.HasValue) + { + document["lastModified"] = LastModified.Value.UtcDateTime; + } + + return document; + } + + public bool Matches(StellaOps.Feedser.Storage.Mongo.Documents.DocumentRecord document) + { + if (document is null) + { + return false; + } + + if (!string.Equals(document.Etag, ETag, StringComparison.Ordinal)) + { + return false; + } + + if (LastModified.HasValue && document.LastModified.HasValue) + { + return LastModified.Value.UtcDateTime == document.LastModified.Value.UtcDateTime; + } + + return !LastModified.HasValue && !document.LastModified.HasValue; + } +} diff --git a/src/StellaOps.Feedser.Source.Distro.Debian/Internal/DebianHtmlParser.cs b/src/StellaOps.Feedser.Source.Distro.Debian/Internal/DebianHtmlParser.cs new file mode 100644 index 00000000..34b22a25 --- /dev/null +++ b/src/StellaOps.Feedser.Source.Distro.Debian/Internal/DebianHtmlParser.cs @@ -0,0 +1,326 @@ +using System; +using System.Collections.Generic; +using System.Globalization; +using System.Linq; +using AngleSharp.Html.Dom; +using AngleSharp.Html.Parser; + +namespace StellaOps.Feedser.Source.Distro.Debian.Internal; + +internal static class DebianHtmlParser +{ + public static DebianAdvisoryDto Parse(string html, DebianDetailMetadata metadata) + { + ArgumentException.ThrowIfNullOrEmpty(html); + ArgumentNullException.ThrowIfNull(metadata); + + var parser = new HtmlParser(); + var document = parser.ParseDocument(html); + + var description = ExtractDescription(document) ?? metadata.Title; + var references = ExtractReferences(document, metadata); + var packages = ExtractPackages(document, metadata.SourcePackage, metadata.Published); + + return new DebianAdvisoryDto( + metadata.AdvisoryId, + metadata.SourcePackage, + metadata.Title, + description, + metadata.CveIds, + packages, + references); + } + + private static string? ExtractDescription(IHtmlDocument document) + { + foreach (var table in document.QuerySelectorAll("table")) + { + if (table is not IHtmlTableElement tableElement) + { + continue; + } + + foreach (var row in tableElement.Rows) + { + if (row.Cells.Length < 2) + { + continue; + } + + var header = row.Cells[0].TextContent?.Trim(); + if (string.Equals(header, "Description", StringComparison.OrdinalIgnoreCase)) + { + return NormalizeWhitespace(row.Cells[1].TextContent); + } + } + + // Only the first table contains the metadata rows we need. + break; + } + + return null; + } + + private static IReadOnlyList ExtractReferences(IHtmlDocument document, DebianDetailMetadata metadata) + { + var references = new List(); + var seen = new HashSet(StringComparer.OrdinalIgnoreCase); + + // Add canonical Debian advisory page. + var canonical = new Uri($"https://www.debian.org/security/{metadata.AdvisoryId.ToLowerInvariant()}"); + references.Add(new DebianReferenceDto(canonical.ToString(), "advisory", metadata.Title)); + seen.Add(canonical.ToString()); + + foreach (var link in document.QuerySelectorAll("a")) + { + var href = link.GetAttribute("href"); + if (string.IsNullOrWhiteSpace(href)) + { + continue; + } + + string resolved; + if (Uri.TryCreate(href, UriKind.Absolute, out var absolute)) + { + resolved = absolute.ToString(); + } + else if (Uri.TryCreate(metadata.DetailUri, href, out var relative)) + { + resolved = relative.ToString(); + } + else + { + continue; + } + + if (!seen.Add(resolved)) + { + continue; + } + + var text = NormalizeWhitespace(link.TextContent); + string? kind = null; + if (text.StartsWith("CVE-", StringComparison.OrdinalIgnoreCase)) + { + kind = "cve"; + } + else if (resolved.Contains("debian.org/security", StringComparison.OrdinalIgnoreCase)) + { + kind = "advisory"; + } + + references.Add(new DebianReferenceDto(resolved, kind, text)); + } + + return references; + } + + private static IReadOnlyList ExtractPackages(IHtmlDocument document, string defaultPackage, DateTimeOffset published) + { + var table = FindPackagesTable(document); + if (table is null) + { + return Array.Empty(); + } + + var accumulators = new Dictionary(StringComparer.OrdinalIgnoreCase); + string currentPackage = defaultPackage; + + foreach (var body in table.Bodies) + { + foreach (var row in body.Rows) + { + if (row.Cells.Length < 4) + { + continue; + } + + var packageCell = NormalizeWhitespace(row.Cells[0].TextContent); + if (!string.IsNullOrWhiteSpace(packageCell)) + { + currentPackage = ExtractPackageName(packageCell); + } + + if (string.IsNullOrWhiteSpace(currentPackage)) + { + continue; + } + + var releaseRaw = NormalizeWhitespace(row.Cells[1].TextContent); + var versionRaw = NormalizeWhitespace(row.Cells[2].TextContent); + var statusRaw = NormalizeWhitespace(row.Cells[3].TextContent); + if (string.IsNullOrWhiteSpace(releaseRaw)) + { + continue; + } + + var release = NormalizeRelease(releaseRaw); + var key = $"{currentPackage}|{release}"; + if (!accumulators.TryGetValue(key, out var accumulator)) + { + accumulator = new PackageAccumulator(currentPackage, release, published); + accumulators[key] = accumulator; + } + + accumulator.Apply(statusRaw, versionRaw); + } + } + + return accumulators.Values + .Where(static acc => acc.ShouldEmit) + .Select(static acc => acc.ToDto()) + .OrderBy(static dto => dto.Release, StringComparer.OrdinalIgnoreCase) + .ThenBy(static dto => dto.Package, StringComparer.OrdinalIgnoreCase) + .ToArray(); + } + + private static IHtmlTableElement? FindPackagesTable(IHtmlDocument document) + { + foreach (var table in document.QuerySelectorAll("table")) + { + if (table is not IHtmlTableElement tableElement) + { + continue; + } + + var header = tableElement.Rows.FirstOrDefault(); + if (header is null || header.Cells.Length < 4) + { + continue; + } + + var firstHeader = NormalizeWhitespace(header.Cells[0].TextContent); + var secondHeader = NormalizeWhitespace(header.Cells[1].TextContent); + var thirdHeader = NormalizeWhitespace(header.Cells[2].TextContent); + if (string.Equals(firstHeader, "Source Package", StringComparison.OrdinalIgnoreCase) + && string.Equals(secondHeader, "Release", StringComparison.OrdinalIgnoreCase) + && string.Equals(thirdHeader, "Version", StringComparison.OrdinalIgnoreCase)) + { + return tableElement; + } + } + + return null; + } + + private static string NormalizeRelease(string release) + { + var trimmed = release.Trim(); + var parenthesisIndex = trimmed.IndexOf('('); + if (parenthesisIndex > 0) + { + trimmed = trimmed[..parenthesisIndex].Trim(); + } + + return trimmed; + } + + private static string ExtractPackageName(string value) + { + var trimmed = value.Split(' ', StringSplitOptions.TrimEntries | StringSplitOptions.RemoveEmptyEntries).FirstOrDefault(); + if (string.IsNullOrWhiteSpace(trimmed)) + { + return value.Trim(); + } + + if (trimmed.EndsWith(")", StringComparison.Ordinal) && trimmed.Contains('(')) + { + trimmed = trimmed[..trimmed.IndexOf('(')]; + } + + return trimmed.Trim(); + } + + private static string NormalizeWhitespace(string value) + => string.IsNullOrWhiteSpace(value) + ? string.Empty + : string.Join(' ', value.Split((char[]?)null, StringSplitOptions.RemoveEmptyEntries | StringSplitOptions.TrimEntries)); + + private sealed class PackageAccumulator + { + private readonly DateTimeOffset _published; + + public PackageAccumulator(string package, string release, DateTimeOffset published) + { + Package = package; + Release = release; + _published = published; + Status = "unknown"; + } + + public string Package { get; } + + public string Release { get; } + + public string Status { get; private set; } + + public string? IntroducedVersion { get; private set; } + + public string? FixedVersion { get; private set; } + + public string? LastAffectedVersion { get; private set; } + + public bool ShouldEmit => + !string.Equals(Status, "not_affected", StringComparison.OrdinalIgnoreCase) + || IntroducedVersion is not null + || FixedVersion is not null; + + public void Apply(string statusRaw, string versionRaw) + { + var status = statusRaw.ToLowerInvariant(); + var version = string.IsNullOrWhiteSpace(versionRaw) ? null : versionRaw.Trim(); + + if (status.Contains("fixed", StringComparison.OrdinalIgnoreCase)) + { + FixedVersion = version; + if (!string.Equals(Status, "open", StringComparison.OrdinalIgnoreCase)) + { + Status = "resolved"; + } + + return; + } + + if (status.Contains("vulnerable", StringComparison.OrdinalIgnoreCase) + || status.Contains("open", StringComparison.OrdinalIgnoreCase)) + { + IntroducedVersion ??= version; + if (!string.Equals(Status, "resolved", StringComparison.OrdinalIgnoreCase)) + { + Status = "open"; + } + + LastAffectedVersion = null; + return; + } + + if (status.Contains("not affected", StringComparison.OrdinalIgnoreCase) + || status.Contains("not vulnerable", StringComparison.OrdinalIgnoreCase)) + { + Status = "not_affected"; + IntroducedVersion = null; + FixedVersion = null; + LastAffectedVersion = null; + return; + } + + if (status.Contains("end-of-life", StringComparison.OrdinalIgnoreCase) || status.Contains("end of life", StringComparison.OrdinalIgnoreCase)) + { + Status = "end_of_life"; + return; + } + + Status = statusRaw; + } + + public DebianPackageStateDto ToDto() + => new( + Package: Package, + Release: Release, + Status: Status, + IntroducedVersion: IntroducedVersion, + FixedVersion: FixedVersion, + LastAffectedVersion: LastAffectedVersion, + Published: _published); + } +} diff --git a/src/StellaOps.Feedser.Source.Distro.Debian/Internal/DebianListEntry.cs b/src/StellaOps.Feedser.Source.Distro.Debian/Internal/DebianListEntry.cs new file mode 100644 index 00000000..81e708fa --- /dev/null +++ b/src/StellaOps.Feedser.Source.Distro.Debian/Internal/DebianListEntry.cs @@ -0,0 +1,11 @@ +using System; +using System.Collections.Generic; + +namespace StellaOps.Feedser.Source.Distro.Debian.Internal; + +internal sealed record DebianListEntry( + string AdvisoryId, + DateTimeOffset Published, + string Title, + string SourcePackage, + IReadOnlyList CveIds); diff --git a/src/StellaOps.Feedser.Source.Distro.Debian/Internal/DebianListParser.cs b/src/StellaOps.Feedser.Source.Distro.Debian/Internal/DebianListParser.cs new file mode 100644 index 00000000..3e22e4b0 --- /dev/null +++ b/src/StellaOps.Feedser.Source.Distro.Debian/Internal/DebianListParser.cs @@ -0,0 +1,107 @@ +using System; +using System.Collections.Generic; +using System.Globalization; +using System.Text.RegularExpressions; + +namespace StellaOps.Feedser.Source.Distro.Debian.Internal; + +internal static class DebianListParser +{ + private static readonly Regex HeaderRegex = new("^\\[(?[^\\]]+)\\]\\s+(?DSA-\\d{4,}-\\d+)\\s+(?.+)$", RegexOptions.Compiled); + private static readonly Regex CveRegex = new("CVE-\\d{4}-\\d{3,7}", RegexOptions.IgnoreCase | RegexOptions.Compiled); + + public static IReadOnlyList<DebianListEntry> Parse(string? content) + { + if (string.IsNullOrWhiteSpace(content)) + { + return Array.Empty<DebianListEntry>(); + } + + var entries = new List<DebianListEntry>(); + var currentCves = new HashSet<string>(StringComparer.OrdinalIgnoreCase); + DateTimeOffset currentDate = default; + string? currentId = null; + string? currentTitle = null; + string? currentPackage = null; + + foreach (var rawLine in content.Split('\n')) + { + var line = rawLine.TrimEnd('\r'); + if (string.IsNullOrWhiteSpace(line)) + { + continue; + } + + if (line[0] == '[') + { + if (currentId is not null && currentTitle is not null && currentPackage is not null) + { + entries.Add(new DebianListEntry( + currentId, + currentDate, + currentTitle, + currentPackage, + currentCves.Count == 0 ? Array.Empty<string>() : new List<string>(currentCves))); + } + + currentCves.Clear(); + currentId = null; + currentTitle = null; + currentPackage = null; + + var match = HeaderRegex.Match(line); + if (!match.Success) + { + continue; + } + + if (!DateTimeOffset.TryParseExact( + match.Groups["date"].Value, + new[] { "dd MMM yyyy", "d MMM yyyy" }, + CultureInfo.InvariantCulture, + DateTimeStyles.AssumeUniversal | DateTimeStyles.AdjustToUniversal, + out currentDate)) + { + continue; + } + + currentId = match.Groups["id"].Value.Trim(); + currentTitle = match.Groups["title"].Value.Trim(); + + var separatorIndex = currentTitle.IndexOf(" - ", StringComparison.Ordinal); + currentPackage = separatorIndex > 0 + ? currentTitle[..separatorIndex].Trim() + : currentTitle.Split(' ', StringSplitOptions.RemoveEmptyEntries | StringSplitOptions.TrimEntries).FirstOrDefault(); + if (string.IsNullOrWhiteSpace(currentPackage)) + { + currentPackage = currentId; + } + + continue; + } + + if (line[0] == '{') + { + foreach (Match match in CveRegex.Matches(line)) + { + if (match.Success && !string.IsNullOrWhiteSpace(match.Value)) + { + currentCves.Add(match.Value.ToUpperInvariant()); + } + } + } + } + + if (currentId is not null && currentTitle is not null && currentPackage is not null) + { + entries.Add(new DebianListEntry( + currentId, + currentDate, + currentTitle, + currentPackage, + currentCves.Count == 0 ? Array.Empty<string>() : new List<string>(currentCves))); + } + + return entries; + } +} diff --git a/src/StellaOps.Feedser.Source.Distro.Debian/Internal/DebianMapper.cs b/src/StellaOps.Feedser.Source.Distro.Debian/Internal/DebianMapper.cs new file mode 100644 index 00000000..91ebcc01 --- /dev/null +++ b/src/StellaOps.Feedser.Source.Distro.Debian/Internal/DebianMapper.cs @@ -0,0 +1,266 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using StellaOps.Feedser.Models; +using StellaOps.Feedser.Normalization.Distro; +using StellaOps.Feedser.Source.Common; +using StellaOps.Feedser.Storage.Mongo.Documents; + +namespace StellaOps.Feedser.Source.Distro.Debian.Internal; + +internal static class DebianMapper +{ + public static Advisory Map( + DebianAdvisoryDto dto, + DocumentRecord document, + DateTimeOffset recordedAt) + { + ArgumentNullException.ThrowIfNull(dto); + ArgumentNullException.ThrowIfNull(document); + + var aliases = BuildAliases(dto); + var references = BuildReferences(dto, recordedAt); + var affectedPackages = BuildAffectedPackages(dto, recordedAt); + + var fetchProvenance = new AdvisoryProvenance( + DebianConnectorPlugin.SourceName, + "document", + document.Uri, + document.FetchedAt.ToUniversalTime()); + + var mappingProvenance = new AdvisoryProvenance( + DebianConnectorPlugin.SourceName, + "mapping", + dto.AdvisoryId, + recordedAt); + + return new Advisory( + advisoryKey: dto.AdvisoryId, + title: dto.Title ?? dto.AdvisoryId, + summary: dto.Description, + language: "en", + published: dto.Packages.Select(p => p.Published).Where(p => p.HasValue).Select(p => p!.Value).Cast<DateTimeOffset?>().DefaultIfEmpty(null).Min(), + modified: recordedAt, + severity: null, + exploitKnown: false, + aliases: aliases, + references: references, + affectedPackages: affectedPackages, + cvssMetrics: Array.Empty<CvssMetric>(), + provenance: new[] { fetchProvenance, mappingProvenance }); + } + + private static string[] BuildAliases(DebianAdvisoryDto dto) + { + var aliases = new HashSet<string>(StringComparer.OrdinalIgnoreCase); + if (!string.IsNullOrWhiteSpace(dto.AdvisoryId)) + { + aliases.Add(dto.AdvisoryId.Trim()); + } + + foreach (var cve in dto.CveIds ?? Array.Empty<string>()) + { + if (!string.IsNullOrWhiteSpace(cve)) + { + aliases.Add(cve.Trim()); + } + } + + return aliases.OrderBy(a => a, StringComparer.OrdinalIgnoreCase).ToArray(); + } + + private static AdvisoryReference[] BuildReferences(DebianAdvisoryDto dto, DateTimeOffset recordedAt) + { + if (dto.References is null || dto.References.Count == 0) + { + return Array.Empty<AdvisoryReference>(); + } + + var references = new List<AdvisoryReference>(); + foreach (var reference in dto.References) + { + if (string.IsNullOrWhiteSpace(reference.Url)) + { + continue; + } + + try + { + var provenance = new AdvisoryProvenance( + DebianConnectorPlugin.SourceName, + "reference", + reference.Url, + recordedAt); + + references.Add(new AdvisoryReference( + reference.Url, + NormalizeReferenceKind(reference.Kind), + reference.Kind, + reference.Title, + provenance)); + } + catch (ArgumentException) + { + // Ignore malformed URLs while keeping the rest of the advisory intact. + } + } + + return references.Count == 0 + ? Array.Empty<AdvisoryReference>() + : references + .OrderBy(r => r.Url, StringComparer.OrdinalIgnoreCase) + .ToArray(); + } + + private static string? NormalizeReferenceKind(string? value) + { + if (string.IsNullOrWhiteSpace(value)) + { + return null; + } + + return value.Trim().ToLowerInvariant() switch + { + "advisory" or "dsa" => "advisory", + "cve" => "cve", + "patch" => "patch", + _ => null, + }; + } + + private static AdvisoryProvenance BuildPackageProvenance(DebianPackageStateDto package, DateTimeOffset recordedAt) + => new(DebianConnectorPlugin.SourceName, "affected", $"{package.Package}:{package.Release}", recordedAt); + + private static IReadOnlyList<AffectedPackage> BuildAffectedPackages(DebianAdvisoryDto dto, DateTimeOffset recordedAt) + { + if (dto.Packages is null || dto.Packages.Count == 0) + { + return Array.Empty<AffectedPackage>(); + } + + var packages = new List<AffectedPackage>(dto.Packages.Count); + foreach (var package in dto.Packages) + { + if (string.IsNullOrWhiteSpace(package.Package)) + { + continue; + } + + var provenance = new[] { BuildPackageProvenance(package, recordedAt) }; + var ranges = BuildVersionRanges(package, recordedAt); + + packages.Add(new AffectedPackage( + AffectedPackageTypes.Deb, + identifier: package.Package.Trim(), + platform: package.Release, + versionRanges: ranges, + statuses: Array.Empty<AffectedPackageStatus>(), + provenance: provenance)); + } + + return packages; + } + + private static IReadOnlyList<AffectedVersionRange> BuildVersionRanges(DebianPackageStateDto package, DateTimeOffset recordedAt) + { + var provenance = new AdvisoryProvenance( + DebianConnectorPlugin.SourceName, + "range", + $"{package.Package}:{package.Release}", + recordedAt); + + var introduced = package.IntroducedVersion; + var fixedVersion = package.FixedVersion; + var lastAffected = package.LastAffectedVersion; + + if (string.IsNullOrWhiteSpace(introduced) && string.IsNullOrWhiteSpace(fixedVersion) && string.IsNullOrWhiteSpace(lastAffected)) + { + return Array.Empty<AffectedVersionRange>(); + } + + var extensions = new Dictionary<string, string>(StringComparer.Ordinal) + { + ["debian.release"] = package.Release, + ["debian.status"] = package.Status + }; + + AddExtension(extensions, "debian.introduced", introduced); + AddExtension(extensions, "debian.fixed", fixedVersion); + AddExtension(extensions, "debian.lastAffected", lastAffected); + + var primitives = BuildEvrPrimitives(introduced, fixedVersion, lastAffected); + return new[] + { + new AffectedVersionRange( + rangeKind: "evr", + introducedVersion: introduced, + fixedVersion: fixedVersion, + lastAffectedVersion: lastAffected, + rangeExpression: BuildRangeExpression(introduced, fixedVersion, lastAffected), + provenance: provenance, + primitives: primitives is null && extensions.Count == 0 + ? null + : new RangePrimitives( + SemVer: null, + Nevra: null, + Evr: primitives, + VendorExtensions: extensions.Count == 0 ? null : extensions)) + }; + } + + private static EvrPrimitive? BuildEvrPrimitives(string? introduced, string? fixedVersion, string? lastAffected) + { + var introducedComponent = ParseEvr(introduced); + var fixedComponent = ParseEvr(fixedVersion); + var lastAffectedComponent = ParseEvr(lastAffected); + + if (introducedComponent is null && fixedComponent is null && lastAffectedComponent is null) + { + return null; + } + + return new EvrPrimitive(introducedComponent, fixedComponent, lastAffectedComponent); + } + + private static EvrComponent? ParseEvr(string? value) + { + if (!DebianEvr.TryParse(value, out var evr) || evr is null) + { + return null; + } + + return new EvrComponent( + evr.Epoch, + evr.Version, + evr.Revision.Length == 0 ? null : evr.Revision); + } + + private static string? BuildRangeExpression(string? introduced, string? fixedVersion, string? lastAffected) + { + var parts = new List<string>(); + if (!string.IsNullOrWhiteSpace(introduced)) + { + parts.Add($"introduced:{introduced.Trim()}"); + } + + if (!string.IsNullOrWhiteSpace(fixedVersion)) + { + parts.Add($"fixed:{fixedVersion.Trim()}"); + } + + if (!string.IsNullOrWhiteSpace(lastAffected)) + { + parts.Add($"last:{lastAffected.Trim()}"); + } + + return parts.Count == 0 ? null : string.Join(" ", parts); + } + + private static void AddExtension(IDictionary<string, string> extensions, string key, string? value) + { + if (!string.IsNullOrWhiteSpace(value)) + { + extensions[key] = value.Trim(); + } + } +} diff --git a/src/StellaOps.Feedser.Source.Distro.Debian/Jobs.cs b/src/StellaOps.Feedser.Source.Distro.Debian/Jobs.cs new file mode 100644 index 00000000..0c770787 --- /dev/null +++ b/src/StellaOps.Feedser.Source.Distro.Debian/Jobs.cs @@ -0,0 +1,46 @@ +using System; +using System.Threading; +using System.Threading.Tasks; +using StellaOps.Feedser.Core.Jobs; + +namespace StellaOps.Feedser.Source.Distro.Debian; + +internal static class DebianJobKinds +{ + public const string Fetch = "source:debian:fetch"; + public const string Parse = "source:debian:parse"; + public const string Map = "source:debian:map"; +} + +internal sealed class DebianFetchJob : IJob +{ + private readonly DebianConnector _connector; + + public DebianFetchJob(DebianConnector connector) + => _connector = connector ?? throw new ArgumentNullException(nameof(connector)); + + public Task ExecuteAsync(JobExecutionContext context, CancellationToken cancellationToken) + => _connector.FetchAsync(context.Services, cancellationToken); +} + +internal sealed class DebianParseJob : IJob +{ + private readonly DebianConnector _connector; + + public DebianParseJob(DebianConnector connector) + => _connector = connector ?? throw new ArgumentNullException(nameof(connector)); + + public Task ExecuteAsync(JobExecutionContext context, CancellationToken cancellationToken) + => _connector.ParseAsync(context.Services, cancellationToken); +} + +internal sealed class DebianMapJob : IJob +{ + private readonly DebianConnector _connector; + + public DebianMapJob(DebianConnector connector) + => _connector = connector ?? throw new ArgumentNullException(nameof(connector)); + + public Task ExecuteAsync(JobExecutionContext context, CancellationToken cancellationToken) + => _connector.MapAsync(context.Services, cancellationToken); +} diff --git a/src/StellaOps.Feedser.Source.Distro.Debian/StellaOps.Feedser.Source.Distro.Debian.csproj b/src/StellaOps.Feedser.Source.Distro.Debian/StellaOps.Feedser.Source.Distro.Debian.csproj new file mode 100644 index 00000000..96165c66 --- /dev/null +++ b/src/StellaOps.Feedser.Source.Distro.Debian/StellaOps.Feedser.Source.Distro.Debian.csproj @@ -0,0 +1,17 @@ +<Project Sdk="Microsoft.NET.Sdk"> + + <PropertyGroup> + <TargetFramework>net10.0</TargetFramework> + <ImplicitUsings>enable</ImplicitUsings> + <Nullable>enable</Nullable> + </PropertyGroup> + + <ItemGroup> + <ProjectReference Include="../StellaOps.Plugin/StellaOps.Plugin.csproj" /> + + <ProjectReference Include="../StellaOps.Feedser.Source.Common/StellaOps.Feedser.Source.Common.csproj" /> + <ProjectReference Include="../StellaOps.Feedser.Models/StellaOps.Feedser.Models.csproj" /> + <ProjectReference Include="../StellaOps.Feedser.Normalization/StellaOps.Feedser.Normalization.csproj" /> + <ProjectReference Include="../StellaOps.Feedser.Storage.Mongo/StellaOps.Feedser.Storage.Mongo.csproj" /> + </ItemGroup> +</Project> diff --git a/src/StellaOps.Feedser.Source.Distro.RedHat.Tests/RedHat/Fixtures/csaf-rhsa-2025-0001.json b/src/StellaOps.Feedser.Source.Distro.RedHat.Tests/RedHat/Fixtures/csaf-rhsa-2025-0001.json new file mode 100644 index 00000000..a6871b49 --- /dev/null +++ b/src/StellaOps.Feedser.Source.Distro.RedHat.Tests/RedHat/Fixtures/csaf-rhsa-2025-0001.json @@ -0,0 +1,95 @@ +{ + "document": { + "aggregate_severity": { + "text": "Important" + }, + "lang": "en", + "notes": [ + { + "category": "summary", + "text": "An update fixes a critical kernel issue." + } + ], + "references": [ + { + "category": "self", + "summary": "RHSA advisory", + "url": "https://access.redhat.com/errata/RHSA-2025:0001" + } + ], + "title": "Red Hat Security Advisory: Example kernel update", + "tracking": { + "id": "RHSA-2025:0001", + "initial_release_date": "2025-10-02T00:00:00+00:00", + "current_release_date": "2025-10-03T00:00:00+00:00" + } + }, + "product_tree": { + "branches": [ + { + "category": "product_family", + "branches": [ + { + "category": "product_name", + "product": { + "name": "Red Hat Enterprise Linux 8", + "product_id": "8Base-RHEL-8", + "product_identification_helper": { + "cpe": "cpe:/o:redhat:enterprise_linux:8" + } + } + } + ] + }, + { + "category": "product_release", + "branches": [ + { + "category": "product_version", + "product": { + "name": "kernel-0:4.18.0-513.5.1.el8.x86_64", + "product_id": "kernel-0:4.18.0-513.5.1.el8.x86_64", + "product_identification_helper": { + "purl": "pkg:rpm/redhat/kernel@4.18.0-513.5.1.el8?arch=x86_64" + } + } + } + ] + } + ] + }, + "vulnerabilities": [ + { + "cve": "CVE-2025-0001", + "references": [ + { + "category": "external", + "summary": "CVE record", + "url": "https://www.cve.org/CVERecord?id=CVE-2025-0001" + } + ], + "scores": [ + { + "cvss_v3": { + "baseScore": 9.8, + "baseSeverity": "CRITICAL", + "vectorString": "CVSS:3.1/AV:N/AC:L/PR:N/UI:N/S:U/C:H/I:H/A:H", + "version": "3.1" + } + } + ], + "product_status": { + "fixed": [ + "8Base-RHEL-8:kernel-0:4.18.0-513.5.1.el8.x86_64" + ], + "first_fixed": [ + "8Base-RHEL-8:kernel-0:4.18.0-513.5.1.el8.x86_64" + ], + "known_affected": [ + "8Base-RHEL-8", + "8Base-RHEL-8:kernel-0:4.18.0-500.1.0.el8.x86_64" + ] + } + } + ] +} diff --git a/src/StellaOps.Feedser.Source.Distro.RedHat.Tests/RedHat/Fixtures/csaf-rhsa-2025-0002.json b/src/StellaOps.Feedser.Source.Distro.RedHat.Tests/RedHat/Fixtures/csaf-rhsa-2025-0002.json new file mode 100644 index 00000000..6071ce34 --- /dev/null +++ b/src/StellaOps.Feedser.Source.Distro.RedHat.Tests/RedHat/Fixtures/csaf-rhsa-2025-0002.json @@ -0,0 +1,82 @@ +{ + "document": { + "aggregate_severity": { + "text": "Moderate" + }, + "lang": "en", + "notes": [ + { + "category": "summary", + "text": "Second advisory covering unaffected packages." + } + ], + "references": [ + { + "category": "self", + "summary": "RHSA advisory", + "url": "https://access.redhat.com/errata/RHSA-2025:0002" + } + ], + "title": "Red Hat Security Advisory: Follow-up kernel status", + "tracking": { + "id": "RHSA-2025:0002", + "initial_release_date": "2025-10-05T12:00:00+00:00", + "current_release_date": "2025-10-05T12:00:00+00:00" + } + }, + "product_tree": { + "branches": [ + { + "category": "product_family", + "branches": [ + { + "category": "product_name", + "product": { + "name": "Red Hat Enterprise Linux 9", + "product_id": "9Base-RHEL-9", + "product_identification_helper": { + "cpe": "cpe:/o:redhat:enterprise_linux:9" + } + } + } + ] + }, + { + "category": "product_release", + "branches": [ + { + "category": "product_version", + "product": { + "name": "kernel-0:5.14.0-400.el9.x86_64", + "product_id": "kernel-0:5.14.0-400.el9.x86_64", + "product_identification_helper": { + "purl": "pkg:rpm/redhat/kernel@5.14.0-400.el9?arch=x86_64" + } + } + } + ] + } + ] + }, + "vulnerabilities": [ + { + "cve": "CVE-2025-0002", + "references": [ + { + "category": "external", + "summary": "CVE record", + "url": "https://www.cve.org/CVERecord?id=CVE-2025-0002" + } + ], + "product_status": { + "known_not_affected": [ + "9Base-RHEL-9", + "9Base-RHEL-9:kernel-0:5.14.0-400.el9.x86_64" + ], + "under_investigation": [ + "9Base-RHEL-9:kernel-0:5.14.0-401.el9.x86_64" + ] + } + } + ] +} diff --git a/src/StellaOps.Feedser.Source.Distro.RedHat.Tests/RedHat/Fixtures/csaf-rhsa-2025-0003.json b/src/StellaOps.Feedser.Source.Distro.RedHat.Tests/RedHat/Fixtures/csaf-rhsa-2025-0003.json new file mode 100644 index 00000000..0f33308d --- /dev/null +++ b/src/StellaOps.Feedser.Source.Distro.RedHat.Tests/RedHat/Fixtures/csaf-rhsa-2025-0003.json @@ -0,0 +1,93 @@ +{ + "document": { + "aggregate_severity": { + "text": "Important" + }, + "lang": "en", + "notes": [ + { + "category": "summary", + "text": "Advisory with mixed reference sources to verify dedupe ordering." + } + ], + "references": [ + { + "category": "self", + "summary": "Primary advisory", + "url": "https://access.redhat.com/errata/RHSA-2025:0003" + }, + { + "category": "self", + "summary": "", + "url": "https://access.redhat.com/errata/RHSA-2025:0003" + }, + { + "category": "mitigation", + "summary": "Knowledge base guidance", + "url": "https://access.redhat.com/solutions/999999" + } + ], + "title": "Red Hat Security Advisory: Reference dedupe validation", + "tracking": { + "id": "RHSA-2025:0003", + "initial_release_date": "2025-10-06T09:00:00+00:00", + "current_release_date": "2025-10-06T09:00:00+00:00" + } + }, + "product_tree": { + "branches": [ + { + "category": "product_family", + "branches": [ + { + "category": "product_name", + "product": { + "name": "Red Hat Enterprise Linux 9", + "product_id": "9Base-RHEL-9", + "product_identification_helper": { + "cpe": "cpe:/o:redhat:enterprise_linux:9" + } + } + } + ] + } + ] + }, + "vulnerabilities": [ + { + "cve": "CVE-2025-0003", + "references": [ + { + "category": "external", + "summary": "CVE record", + "url": "https://www.cve.org/CVERecord?id=CVE-2025-0003" + }, + { + "category": "external", + "summary": "", + "url": "https://www.cve.org/CVERecord?id=CVE-2025-0003" + }, + { + "category": "exploit", + "summary": "Exploit tracking", + "url": "https://bugzilla.redhat.com/show_bug.cgi?id=2222222" + } + ], + "scores": [ + { + "cvss_v3": { + "baseScore": 7.5, + "baseSeverity": "HIGH", + "vectorString": "CVSS:3.1/AV:N/AC:L/PR:N/UI:N/S:U/C:H/I:N/A:N", + "version": "3.1" + } + } + ], + "product_status": { + "known_affected": [ + "9Base-RHEL-9" + ] + } + } + ] +} diff --git a/src/StellaOps.Feedser.Source.Distro.RedHat.Tests/RedHat/Fixtures/rhsa-2025-0001.snapshot.json b/src/StellaOps.Feedser.Source.Distro.RedHat.Tests/RedHat/Fixtures/rhsa-2025-0001.snapshot.json new file mode 100644 index 00000000..da219872 --- /dev/null +++ b/src/StellaOps.Feedser.Source.Distro.RedHat.Tests/RedHat/Fixtures/rhsa-2025-0001.snapshot.json @@ -0,0 +1,149 @@ +{ + "advisoryKey": "RHSA-2025:0001", + "affectedPackages": [ + { + "identifier": "cpe:2.3:o:redhat:enterprise_linux:8:*:*:*:*:*:*:*", + "platform": "Red Hat Enterprise Linux 8", + "provenance": [ + { + "fieldMask": [], + "kind": "oval", + "recordedAt": "2025-10-05T00:00:00+00:00", + "source": "redhat", + "value": "8Base-RHEL-8" + } + ], + "statuses": [ + { + "provenance": { + "fieldMask": [], + "kind": "oval", + "recordedAt": "2025-10-05T00:00:00+00:00", + "source": "redhat", + "value": "8Base-RHEL-8" + }, + "status": "known_affected" + } + ], + "type": "cpe", + "versionRanges": [] + }, + { + "identifier": "kernel-0:4.18.0-513.5.1.el8.x86_64", + "platform": "Red Hat Enterprise Linux 8", + "provenance": [ + { + "fieldMask": [], + "kind": "package.nevra", + "recordedAt": "2025-10-05T00:00:00+00:00", + "source": "redhat", + "value": "kernel-0:4.18.0-513.5.1.el8.x86_64" + } + ], + "statuses": [], + "type": "rpm", + "versionRanges": [ + { + "fixedVersion": "kernel-0:4.18.0-513.5.1.el8.x86_64", + "introducedVersion": null, + "lastAffectedVersion": "kernel-0:4.18.0-500.1.0.el8.x86_64", + "primitives": { + "evr": null, + "hasVendorExtensions": false, + "nevra": { + "fixed": { + "architecture": "x86_64", + "epoch": 0, + "name": "kernel", + "release": "513.5.1.el8", + "version": "4.18.0" + }, + "introduced": null, + "lastAffected": { + "architecture": "x86_64", + "epoch": 0, + "name": "kernel", + "release": "500.1.0.el8", + "version": "4.18.0" + } + }, + "semVer": null, + "vendorExtensions": null + }, + "provenance": { + "fieldMask": [], + "kind": "package.nevra", + "recordedAt": "2025-10-05T00:00:00+00:00", + "source": "redhat", + "value": "kernel-0:4.18.0-513.5.1.el8.x86_64" + }, + "rangeExpression": null, + "rangeKind": "nevra" + } + ] + } + ], + "aliases": [ + "CVE-2025-0001", + "RHSA-2025:0001" + ], + "cvssMetrics": [ + { + "baseScore": 9.8, + "baseSeverity": "critical", + "provenance": { + "fieldMask": [], + "kind": "cvss", + "recordedAt": "2025-10-05T00:00:00+00:00", + "source": "redhat", + "value": "CVE-2025-0001" + }, + "vector": "CVSS:3.1/AV:N/AC:L/PR:N/UI:N/S:U/C:H/I:H/A:H", + "version": "3.1" + } + ], + "exploitKnown": false, + "language": "en", + "modified": "2025-10-03T00:00:00+00:00", + "provenance": [ + { + "fieldMask": [], + "kind": "advisory", + "recordedAt": "2025-10-05T00:00:00+00:00", + "source": "redhat", + "value": "RHSA-2025:0001" + } + ], + "published": "2025-10-02T00:00:00+00:00", + "references": [ + { + "kind": "self", + "provenance": { + "fieldMask": [], + "kind": "reference", + "recordedAt": "2025-10-05T00:00:00+00:00", + "source": "redhat", + "value": "https://access.redhat.com/errata/RHSA-2025:0001" + }, + "sourceTag": null, + "summary": "RHSA advisory", + "url": "https://access.redhat.com/errata/RHSA-2025:0001" + }, + { + "kind": "external", + "provenance": { + "fieldMask": [], + "kind": "reference", + "recordedAt": "2025-10-05T00:00:00+00:00", + "source": "redhat", + "value": "https://www.cve.org/CVERecord?id=CVE-2025-0001" + }, + "sourceTag": null, + "summary": "CVE record", + "url": "https://www.cve.org/CVERecord?id=CVE-2025-0001" + } + ], + "severity": "high", + "summary": "An update fixes a critical kernel issue.", + "title": "Red Hat Security Advisory: Example kernel update" +} \ No newline at end of file diff --git a/src/StellaOps.Feedser.Source.Distro.RedHat.Tests/RedHat/Fixtures/rhsa-2025-0002.snapshot.json b/src/StellaOps.Feedser.Source.Distro.RedHat.Tests/RedHat/Fixtures/rhsa-2025-0002.snapshot.json new file mode 100644 index 00000000..acdd5763 --- /dev/null +++ b/src/StellaOps.Feedser.Source.Distro.RedHat.Tests/RedHat/Fixtures/rhsa-2025-0002.snapshot.json @@ -0,0 +1,118 @@ +{ + "advisoryKey": "RHSA-2025:0002", + "affectedPackages": [ + { + "identifier": "cpe:2.3:o:redhat:enterprise_linux:9:*:*:*:*:*:*:*", + "platform": "Red Hat Enterprise Linux 9", + "provenance": [ + { + "fieldMask": [], + "kind": "oval", + "recordedAt": "2025-10-05T12:00:00+00:00", + "source": "redhat", + "value": "9Base-RHEL-9" + } + ], + "statuses": [ + { + "provenance": { + "fieldMask": [], + "kind": "oval", + "recordedAt": "2025-10-05T12:00:00+00:00", + "source": "redhat", + "value": "9Base-RHEL-9" + }, + "status": "known_not_affected" + }, + { + "provenance": { + "fieldMask": [], + "kind": "oval", + "recordedAt": "2025-10-05T12:00:00+00:00", + "source": "redhat", + "value": "9Base-RHEL-9" + }, + "status": "under_investigation" + } + ], + "type": "cpe", + "versionRanges": [] + }, + { + "identifier": "kernel-0:5.14.0-400.el9.x86_64", + "platform": "Red Hat Enterprise Linux 9", + "provenance": [ + { + "fieldMask": [], + "kind": "package.nevra", + "recordedAt": "2025-10-05T12:00:00+00:00", + "source": "redhat", + "value": "kernel-0:5.14.0-400.el9.x86_64" + } + ], + "statuses": [ + { + "provenance": { + "fieldMask": [], + "kind": "package.nevra", + "recordedAt": "2025-10-05T12:00:00+00:00", + "source": "redhat", + "value": "kernel-0:5.14.0-400.el9.x86_64" + }, + "status": "known_not_affected" + } + ], + "type": "rpm", + "versionRanges": [] + } + ], + "aliases": [ + "CVE-2025-0002", + "RHSA-2025:0002" + ], + "cvssMetrics": [], + "exploitKnown": false, + "language": "en", + "modified": "2025-10-05T12:00:00+00:00", + "provenance": [ + { + "fieldMask": [], + "kind": "advisory", + "recordedAt": "2025-10-05T12:00:00+00:00", + "source": "redhat", + "value": "RHSA-2025:0002" + } + ], + "published": "2025-10-05T12:00:00+00:00", + "references": [ + { + "kind": "self", + "provenance": { + "fieldMask": [], + "kind": "reference", + "recordedAt": "2025-10-05T12:00:00+00:00", + "source": "redhat", + "value": "https://access.redhat.com/errata/RHSA-2025:0002" + }, + "sourceTag": null, + "summary": "RHSA advisory", + "url": "https://access.redhat.com/errata/RHSA-2025:0002" + }, + { + "kind": "external", + "provenance": { + "fieldMask": [], + "kind": "reference", + "recordedAt": "2025-10-05T12:00:00+00:00", + "source": "redhat", + "value": "https://www.cve.org/CVERecord?id=CVE-2025-0002" + }, + "sourceTag": null, + "summary": "CVE record", + "url": "https://www.cve.org/CVERecord?id=CVE-2025-0002" + } + ], + "severity": "medium", + "summary": "Second advisory covering unaffected packages.", + "title": "Red Hat Security Advisory: Follow-up kernel status" +} \ No newline at end of file diff --git a/src/StellaOps.Feedser.Source.Distro.RedHat.Tests/RedHat/Fixtures/rhsa-2025-0003.snapshot.json b/src/StellaOps.Feedser.Source.Distro.RedHat.Tests/RedHat/Fixtures/rhsa-2025-0003.snapshot.json new file mode 100644 index 00000000..9649bae7 --- /dev/null +++ b/src/StellaOps.Feedser.Source.Distro.RedHat.Tests/RedHat/Fixtures/rhsa-2025-0003.snapshot.json @@ -0,0 +1,121 @@ +{ + "advisoryKey": "RHSA-2025:0003", + "affectedPackages": [ + { + "identifier": "cpe:2.3:o:redhat:enterprise_linux:9:*:*:*:*:*:*:*", + "platform": "Red Hat Enterprise Linux 9", + "provenance": [ + { + "fieldMask": [], + "kind": "oval", + "recordedAt": "2025-10-06T09:00:00+00:00", + "source": "redhat", + "value": "9Base-RHEL-9" + } + ], + "statuses": [ + { + "provenance": { + "fieldMask": [], + "kind": "oval", + "recordedAt": "2025-10-06T09:00:00+00:00", + "source": "redhat", + "value": "9Base-RHEL-9" + }, + "status": "known_affected" + } + ], + "type": "cpe", + "versionRanges": [] + } + ], + "aliases": [ + "CVE-2025-0003", + "RHSA-2025:0003" + ], + "cvssMetrics": [ + { + "baseScore": 7.5, + "baseSeverity": "high", + "provenance": { + "fieldMask": [], + "kind": "cvss", + "recordedAt": "2025-10-06T09:00:00+00:00", + "source": "redhat", + "value": "CVE-2025-0003" + }, + "vector": "CVSS:3.1/AV:N/AC:L/PR:N/UI:N/S:U/C:H/I:N/A:N", + "version": "3.1" + } + ], + "exploitKnown": false, + "language": "en", + "modified": "2025-10-06T09:00:00+00:00", + "provenance": [ + { + "fieldMask": [], + "kind": "advisory", + "recordedAt": "2025-10-06T09:00:00+00:00", + "source": "redhat", + "value": "RHSA-2025:0003" + } + ], + "published": "2025-10-06T09:00:00+00:00", + "references": [ + { + "kind": "self", + "provenance": { + "fieldMask": [], + "kind": "reference", + "recordedAt": "2025-10-06T09:00:00+00:00", + "source": "redhat", + "value": "https://access.redhat.com/errata/RHSA-2025:0003" + }, + "sourceTag": null, + "summary": "Primary advisory", + "url": "https://access.redhat.com/errata/RHSA-2025:0003" + }, + { + "kind": "mitigation", + "provenance": { + "fieldMask": [], + "kind": "reference", + "recordedAt": "2025-10-06T09:00:00+00:00", + "source": "redhat", + "value": "https://access.redhat.com/solutions/999999" + }, + "sourceTag": null, + "summary": "Knowledge base guidance", + "url": "https://access.redhat.com/solutions/999999" + }, + { + "kind": "exploit", + "provenance": { + "fieldMask": [], + "kind": "reference", + "recordedAt": "2025-10-06T09:00:00+00:00", + "source": "redhat", + "value": "https://bugzilla.redhat.com/show_bug.cgi?id=2222222" + }, + "sourceTag": null, + "summary": "Exploit tracking", + "url": "https://bugzilla.redhat.com/show_bug.cgi?id=2222222" + }, + { + "kind": "external", + "provenance": { + "fieldMask": [], + "kind": "reference", + "recordedAt": "2025-10-06T09:00:00+00:00", + "source": "redhat", + "value": "https://www.cve.org/CVERecord?id=CVE-2025-0003" + }, + "sourceTag": null, + "summary": "CVE record", + "url": "https://www.cve.org/CVERecord?id=CVE-2025-0003" + } + ], + "severity": "high", + "summary": "Advisory with mixed reference sources to verify dedupe ordering.", + "title": "Red Hat Security Advisory: Reference dedupe validation" +} \ No newline at end of file diff --git a/src/StellaOps.Feedser.Source.Distro.RedHat.Tests/RedHat/Fixtures/summary-page1-repeat.json b/src/StellaOps.Feedser.Source.Distro.RedHat.Tests/RedHat/Fixtures/summary-page1-repeat.json new file mode 100644 index 00000000..9b713b60 --- /dev/null +++ b/src/StellaOps.Feedser.Source.Distro.RedHat.Tests/RedHat/Fixtures/summary-page1-repeat.json @@ -0,0 +1,14 @@ +[ + { + "RHSA": "RHSA-2025:0001", + "severity": "important", + "released_on": "2025-10-03T00:00:00Z", + "resource_url": "https://access.redhat.com/hydra/rest/securitydata/csaf/RHSA-2025:0001.json" + }, + { + "RHSA": "RHSA-2025:0002", + "severity": "moderate", + "released_on": "2025-10-05T12:00:00Z", + "resource_url": "https://access.redhat.com/hydra/rest/securitydata/csaf/RHSA-2025:0002.json" + } +] diff --git a/src/StellaOps.Feedser.Source.Distro.RedHat.Tests/RedHat/Fixtures/summary-page1.json b/src/StellaOps.Feedser.Source.Distro.RedHat.Tests/RedHat/Fixtures/summary-page1.json new file mode 100644 index 00000000..7f158304 --- /dev/null +++ b/src/StellaOps.Feedser.Source.Distro.RedHat.Tests/RedHat/Fixtures/summary-page1.json @@ -0,0 +1,14 @@ +[ + { + "RHSA": "RHSA-2025:0001", + "severity": "important", + "released_on": "2025-10-03T12:00:00Z", + "resource_url": "https://access.redhat.com/hydra/rest/securitydata/csaf/RHSA-2025:0001.json" + }, + { + "RHSA": "RHSA-2025:0002", + "severity": "moderate", + "released_on": "2025-10-05T12:00:00Z", + "resource_url": "https://access.redhat.com/hydra/rest/securitydata/csaf/RHSA-2025:0002.json" + } +] diff --git a/src/StellaOps.Feedser.Source.Distro.RedHat.Tests/RedHat/Fixtures/summary-page2.json b/src/StellaOps.Feedser.Source.Distro.RedHat.Tests/RedHat/Fixtures/summary-page2.json new file mode 100644 index 00000000..80905fb5 --- /dev/null +++ b/src/StellaOps.Feedser.Source.Distro.RedHat.Tests/RedHat/Fixtures/summary-page2.json @@ -0,0 +1,8 @@ +[ + { + "RHSA": "RHSA-2025:0002", + "severity": "moderate", + "released_on": "2025-10-05T12:00:00Z", + "resource_url": "https://access.redhat.com/hydra/rest/securitydata/csaf/RHSA-2025:0002.json" + } +] diff --git a/src/StellaOps.Feedser.Source.Distro.RedHat.Tests/RedHat/Fixtures/summary-page3.json b/src/StellaOps.Feedser.Source.Distro.RedHat.Tests/RedHat/Fixtures/summary-page3.json new file mode 100644 index 00000000..be8e0a74 --- /dev/null +++ b/src/StellaOps.Feedser.Source.Distro.RedHat.Tests/RedHat/Fixtures/summary-page3.json @@ -0,0 +1,8 @@ +[ + { + "RHSA": "RHSA-2025:0003", + "severity": "important", + "released_on": "2025-10-06T09:00:00Z", + "resource_url": "https://access.redhat.com/hydra/rest/securitydata/csaf/RHSA-2025:0003.json" + } +] diff --git a/src/StellaOps.Feedser.Source.Distro.RedHat.Tests/RedHat/RedHatConnectorHarnessTests.cs b/src/StellaOps.Feedser.Source.Distro.RedHat.Tests/RedHat/RedHatConnectorHarnessTests.cs new file mode 100644 index 00000000..463f513c --- /dev/null +++ b/src/StellaOps.Feedser.Source.Distro.RedHat.Tests/RedHat/RedHatConnectorHarnessTests.cs @@ -0,0 +1,123 @@ +using System; +using System.IO; +using System.Linq; +using Microsoft.Extensions.DependencyInjection; +using MongoDB.Bson; +using StellaOps.Feedser.Source.Common.Testing; +using StellaOps.Feedser.Source.Distro.RedHat; +using StellaOps.Feedser.Source.Distro.RedHat.Configuration; +using StellaOps.Feedser.Storage.Mongo; +using StellaOps.Feedser.Storage.Mongo.Advisories; +using StellaOps.Feedser.Testing; +using StellaOps.Feedser.Testing; + +namespace StellaOps.Feedser.Source.Distro.RedHat.Tests; + +[Collection("mongo-fixture")] +public sealed class RedHatConnectorHarnessTests : IAsyncLifetime +{ + private readonly ConnectorTestHarness _harness; + + public RedHatConnectorHarnessTests(MongoIntegrationFixture fixture) + { + _harness = new ConnectorTestHarness(fixture, new DateTimeOffset(2025, 10, 5, 0, 0, 0, TimeSpan.Zero), RedHatOptions.HttpClientName); + } + + [Fact] + public async Task FetchParseMap_WithHarness_ProducesCanonicalAdvisory() + { + await _harness.ResetAsync(); + + var options = new RedHatOptions + { + BaseEndpoint = new Uri("https://access.redhat.com/hydra/rest/securitydata"), + PageSize = 10, + MaxPagesPerFetch = 2, + MaxAdvisoriesPerFetch = 5, + InitialBackfill = TimeSpan.FromDays(1), + Overlap = TimeSpan.Zero, + FetchTimeout = TimeSpan.FromSeconds(30), + UserAgent = "StellaOps.Tests.RedHatHarness/1.0", + }; + + var handler = _harness.Handler; + var timeProvider = _harness.TimeProvider; + + var summaryUri = new Uri("https://access.redhat.com/hydra/rest/securitydata/csaf.json?after=2025-10-04&per_page=10&page=1"); + var summaryUriPost = new Uri("https://access.redhat.com/hydra/rest/securitydata/csaf.json?after=2025-10-05&per_page=10&page=1"); + var summaryUriPostPage2 = new Uri("https://access.redhat.com/hydra/rest/securitydata/csaf.json?after=2025-10-05&per_page=10&page=2"); + var detailUri = new Uri("https://access.redhat.com/hydra/rest/securitydata/csaf/RHSA-2025:0001.json"); + var detailUri2 = new Uri("https://access.redhat.com/hydra/rest/securitydata/csaf/RHSA-2025:0002.json"); + + handler.AddJsonResponse(summaryUri, ReadFixture("summary-page1-repeat.json")); + handler.AddJsonResponse(summaryUriPost, "[]"); + handler.AddJsonResponse(summaryUriPostPage2, "[]"); + handler.AddJsonResponse(detailUri, ReadFixture("csaf-rhsa-2025-0001.json")); + handler.AddJsonResponse(detailUri2, ReadFixture("csaf-rhsa-2025-0002.json")); + + await _harness.EnsureServiceProviderAsync(services => + { + services.AddRedHatConnector(opts => + { + opts.BaseEndpoint = options.BaseEndpoint; + opts.PageSize = options.PageSize; + opts.MaxPagesPerFetch = options.MaxPagesPerFetch; + opts.MaxAdvisoriesPerFetch = options.MaxAdvisoriesPerFetch; + opts.InitialBackfill = options.InitialBackfill; + opts.Overlap = options.Overlap; + opts.FetchTimeout = options.FetchTimeout; + opts.UserAgent = options.UserAgent; + }); + }); + + var provider = _harness.ServiceProvider; + + var stateRepository = provider.GetRequiredService<ISourceStateRepository>(); + await stateRepository.UpsertAsync( + new SourceStateRecord( + RedHatConnectorPlugin.SourceName, + Enabled: true, + Paused: false, + Cursor: new BsonDocument(), + LastSuccess: null, + LastFailure: null, + FailCount: 0, + BackoffUntil: null, + UpdatedAt: timeProvider.GetUtcNow(), + LastFailureReason: null), + CancellationToken.None); + + var connector = new RedHatConnectorPlugin().Create(provider); + + await connector.FetchAsync(provider, CancellationToken.None); + await connector.ParseAsync(provider, CancellationToken.None); + await connector.MapAsync(provider, CancellationToken.None); + + var advisoryStore = provider.GetRequiredService<IAdvisoryStore>(); + var advisories = await advisoryStore.GetRecentAsync(5, CancellationToken.None); + Assert.Equal(2, advisories.Count); + var advisory = advisories.Single(a => string.Equals(a.AdvisoryKey, "RHSA-2025:0001", StringComparison.Ordinal)); + Assert.Equal("high", advisory.Severity); + Assert.Contains(advisory.Aliases, alias => alias == "CVE-2025-0001"); + Assert.Empty(advisory.Provenance.Where(p => p.Source == "redhat" && p.Kind == "fetch")); + + var secondAdvisory = advisories.Single(a => string.Equals(a.AdvisoryKey, "RHSA-2025:0002", StringComparison.Ordinal)); + Assert.Equal("medium", secondAdvisory.Severity, ignoreCase: true); + Assert.Contains(secondAdvisory.Aliases, alias => alias == "CVE-2025-0002"); + + var state = await stateRepository.TryGetAsync(RedHatConnectorPlugin.SourceName, CancellationToken.None); + Assert.NotNull(state); + Assert.True(state!.Cursor.TryGetValue("pendingDocuments", out var pendingDocs) && pendingDocs.AsBsonArray.Count == 0); + Assert.True(state.Cursor.TryGetValue("pendingMappings", out var pendingMappings) && pendingMappings.AsBsonArray.Count == 0); + } + + public Task InitializeAsync() => Task.CompletedTask; + + public Task DisposeAsync() => _harness.ResetAsync(); + + private static string ReadFixture(string filename) + { + var path = Path.Combine(AppContext.BaseDirectory, "Source", "Distro", "RedHat", "Fixtures", filename); + return File.ReadAllText(path); + } +} diff --git a/src/StellaOps.Feedser.Source.Distro.RedHat.Tests/RedHat/RedHatConnectorTests.cs b/src/StellaOps.Feedser.Source.Distro.RedHat.Tests/RedHat/RedHatConnectorTests.cs new file mode 100644 index 00000000..873980c5 --- /dev/null +++ b/src/StellaOps.Feedser.Source.Distro.RedHat.Tests/RedHat/RedHatConnectorTests.cs @@ -0,0 +1,653 @@ +using System; +using System.Globalization; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.Extensions.Configuration; +using System.Text.Json; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Http; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Logging.Abstractions; +using Microsoft.Extensions.Options; +using Microsoft.Extensions.Time.Testing; +using MongoDB.Bson; +using StellaOps.Feedser.Source.Common; +using StellaOps.Feedser.Core.Jobs; +using StellaOps.Feedser.Source.Common.Fetch; +using StellaOps.Feedser.Source.Common.Http; +using StellaOps.Feedser.Source.Common.Testing; +using StellaOps.Feedser.Source.Distro.RedHat; +using StellaOps.Feedser.Source.Distro.RedHat.Configuration; +using StellaOps.Feedser.Source.Distro.RedHat.Internal; +using StellaOps.Feedser.Models; +using StellaOps.Feedser.Storage.Mongo; +using StellaOps.Feedser.Storage.Mongo.Advisories; +using StellaOps.Feedser.Storage.Mongo.Documents; +using StellaOps.Feedser.Storage.Mongo.Dtos; +using StellaOps.Feedser.Testing; +using StellaOps.Plugin; +using Xunit; +using Xunit.Abstractions; + +namespace StellaOps.Feedser.Source.Distro.RedHat.Tests; + +[Collection("mongo-fixture")] +public sealed class RedHatConnectorTests : IAsyncLifetime +{ + private readonly MongoIntegrationFixture _fixture; + private readonly FakeTimeProvider _timeProvider; + private readonly DateTimeOffset _initialNow; + private readonly CannedHttpMessageHandler _handler; + private readonly ITestOutputHelper _output; + private ServiceProvider? _serviceProvider; + private const bool ForceUpdateGoldens = false; + + public RedHatConnectorTests(MongoIntegrationFixture fixture, ITestOutputHelper output) + { + _fixture = fixture; + _initialNow = new DateTimeOffset(2025, 10, 5, 0, 0, 0, TimeSpan.Zero); + _timeProvider = new FakeTimeProvider(_initialNow); + _handler = new CannedHttpMessageHandler(); + _output = output; + } + + [Fact] + public async Task FetchParseMap_ProducesCanonicalAdvisory() + { + await ResetDatabaseAsync(); + + var options = new RedHatOptions + { + BaseEndpoint = new Uri("https://access.redhat.com/hydra/rest/securitydata"), + PageSize = 10, + MaxPagesPerFetch = 2, + MaxAdvisoriesPerFetch = 25, + InitialBackfill = TimeSpan.FromDays(1), + Overlap = TimeSpan.Zero, + FetchTimeout = TimeSpan.FromSeconds(30), + UserAgent = "StellaOps.Tests.RedHat/1.0", + }; + + await EnsureServiceProviderAsync(options); + var provider = _serviceProvider!; + + var configuredOptions = provider.GetRequiredService<IOptions<RedHatOptions>>().Value; + Assert.Equal(10, configuredOptions.PageSize); + Assert.Equal(TimeSpan.FromDays(1), configuredOptions.InitialBackfill); + Assert.Equal(TimeSpan.Zero, configuredOptions.Overlap); + _output.WriteLine($"InitialBackfill configured: {configuredOptions.InitialBackfill}"); + _output.WriteLine($"TimeProvider now: {_timeProvider.GetUtcNow():O}"); + + var summaryUriBackfill = new Uri("https://access.redhat.com/hydra/rest/securitydata/csaf.json?after=2025-10-03&per_page=10&page=1"); + var summaryUri = new Uri("https://access.redhat.com/hydra/rest/securitydata/csaf.json?after=2025-10-04&per_page=10&page=1"); + var summaryUriPost = new Uri("https://access.redhat.com/hydra/rest/securitydata/csaf.json?after=2025-10-05&per_page=10&page=1"); + var summaryUriPostPage2 = new Uri("https://access.redhat.com/hydra/rest/securitydata/csaf.json?after=2025-10-05&per_page=10&page=2"); + var detailUri = new Uri("https://access.redhat.com/hydra/rest/securitydata/csaf/RHSA-2025:0001.json"); + var detailUri2 = new Uri("https://access.redhat.com/hydra/rest/securitydata/csaf/RHSA-2025:0002.json"); + + _output.WriteLine($"Registering summary URI: {summaryUriBackfill}"); + _output.WriteLine($"Registering summary URI (overlap): {summaryUri}"); + _handler.AddJsonResponse(summaryUriBackfill, ReadFixture("summary-page1.json")); + _handler.AddJsonResponse(summaryUri, ReadFixture("summary-page1-repeat.json")); + _handler.AddJsonResponse(summaryUriPost, "[]"); + _handler.AddJsonResponse(summaryUriPostPage2, "[]"); + _handler.AddJsonResponse(detailUri, ReadFixture("csaf-rhsa-2025-0001.json")); + _handler.AddJsonResponse(detailUri2, ReadFixture("csaf-rhsa-2025-0002.json")); + + var stateRepository = provider.GetRequiredService<ISourceStateRepository>(); + await stateRepository.UpsertAsync( + new SourceStateRecord( + RedHatConnectorPlugin.SourceName, + Enabled: true, + Paused: false, + Cursor: new BsonDocument(), + LastSuccess: null, + LastFailure: null, + FailCount: 0, + BackoffUntil: null, + UpdatedAt: _timeProvider.GetUtcNow(), + LastFailureReason: null), + CancellationToken.None); + + var connector = new RedHatConnectorPlugin().Create(provider); + + await connector.FetchAsync(provider, CancellationToken.None); + await connector.ParseAsync(provider, CancellationToken.None); + await connector.MapAsync(provider, CancellationToken.None); + + + foreach (var request in _handler.Requests) + { + _output.WriteLine($"Captured request: {request.Uri}"); + } + + var advisoryStore = provider.GetRequiredService<IAdvisoryStore>(); + var advisories = await advisoryStore.GetRecentAsync(10, CancellationToken.None); + var advisory = advisories.Single(a => string.Equals(a.AdvisoryKey, "RHSA-2025:0001", StringComparison.Ordinal)); + Assert.Equal("red hat security advisory: example kernel update", advisory.Title.ToLowerInvariant()); + Assert.Contains("RHSA-2025:0001", advisory.Aliases); + Assert.Contains("CVE-2025-0001", advisory.Aliases); + Assert.Equal("high", advisory.Severity); + Assert.Equal("en", advisory.Language); + + var rpmPackage = advisory.AffectedPackages.Single(pkg => pkg.Type == AffectedPackageTypes.Rpm); + _output.WriteLine($"RPM statuses count: {rpmPackage.Statuses.Length}"); + _output.WriteLine($"RPM ranges count: {rpmPackage.VersionRanges.Length}"); + foreach (var range in rpmPackage.VersionRanges) + { + _output.WriteLine($"Range fixed={range.FixedVersion}, last={range.LastAffectedVersion}, expr={range.RangeExpression}"); + } + Assert.Equal("kernel-0:4.18.0-513.5.1.el8.x86_64", rpmPackage.Identifier); + var fixedRange = Assert.Single( + rpmPackage.VersionRanges, + range => string.Equals(range.FixedVersion, "kernel-0:4.18.0-513.5.1.el8.x86_64", StringComparison.Ordinal)); + Assert.Equal("kernel-0:4.18.0-500.1.0.el8.x86_64", fixedRange.LastAffectedVersion); + var nevraPrimitive = fixedRange.Primitives?.Nevra; + Assert.NotNull(nevraPrimitive); + Assert.Null(nevraPrimitive!.Introduced); + Assert.Equal("kernel", nevraPrimitive.Fixed?.Name); + + var cpePackage = advisory.AffectedPackages.Single(pkg => pkg.Type == AffectedPackageTypes.Cpe); + Assert.Equal("cpe:2.3:o:redhat:enterprise_linux:8:*:*:*:*:*:*:*", cpePackage.Identifier); + + Assert.Contains(advisory.References, reference => reference.Url == "https://access.redhat.com/errata/RHSA-2025:0001"); + Assert.Contains(advisory.References, reference => reference.Url == "https://www.cve.org/CVERecord?id=CVE-2025-0001"); + + var snapshot = SnapshotSerializer.ToSnapshot(advisory).Replace("\r\n", "\n"); + _output.WriteLine("-- RHSA-2025:0001 snapshot --\n" + snapshot); + var snapshotPath = ProjectFixturePath("rhsa-2025-0001.snapshot.json"); + if (ShouldUpdateGoldens()) + { + File.WriteAllText(snapshotPath, snapshot); + return; + } + + var expectedSnapshot = File.ReadAllText(snapshotPath); + Assert.Equal(NormalizeLineEndings(expectedSnapshot), NormalizeLineEndings(snapshot)); + + var state = await stateRepository.TryGetAsync(RedHatConnectorPlugin.SourceName, CancellationToken.None); + Assert.NotNull(state); + Assert.True(state!.Cursor.TryGetValue("pendingDocuments", out var pendingDocs2) && pendingDocs2.AsBsonArray.Count == 0); + Assert.True(state.Cursor.TryGetValue("pendingMappings", out var pendingMappings2) && pendingMappings2.AsBsonArray.Count == 0); + + const string fetchKind = "source:redhat:fetch"; + const string parseKind = "source:redhat:parse"; + const string mapKind = "source:redhat:map"; + + var schedulerOptions = provider.GetRequiredService<Microsoft.Extensions.Options.IOptions<JobSchedulerOptions>>().Value; + Assert.True(schedulerOptions.Definitions.TryGetValue(fetchKind, out var fetchDefinition)); + Assert.True(schedulerOptions.Definitions.TryGetValue(parseKind, out var parseDefinition)); + Assert.True(schedulerOptions.Definitions.TryGetValue(mapKind, out var mapDefinition)); + + Assert.Equal("RedHatFetchJob", fetchDefinition.JobType.Name); + Assert.Equal(TimeSpan.FromMinutes(12), fetchDefinition.Timeout); + Assert.Equal(TimeSpan.FromMinutes(6), fetchDefinition.LeaseDuration); + Assert.Equal("0,15,30,45 * * * *", fetchDefinition.CronExpression); + Assert.True(fetchDefinition.Enabled); + + Assert.Equal("RedHatParseJob", parseDefinition.JobType.Name); + Assert.Equal(TimeSpan.FromMinutes(15), parseDefinition.Timeout); + Assert.Equal(TimeSpan.FromMinutes(6), parseDefinition.LeaseDuration); + Assert.Equal("5,20,35,50 * * * *", parseDefinition.CronExpression); + Assert.True(parseDefinition.Enabled); + + Assert.Equal("RedHatMapJob", mapDefinition.JobType.Name); + Assert.Equal(TimeSpan.FromMinutes(20), mapDefinition.Timeout); + Assert.Equal(TimeSpan.FromMinutes(6), mapDefinition.LeaseDuration); + Assert.Equal("10,25,40,55 * * * *", mapDefinition.CronExpression); + Assert.True(mapDefinition.Enabled); + + var summaryUriRepeat = new Uri("https://access.redhat.com/hydra/rest/securitydata/csaf.json?after=2025-10-03&per_page=10&page=1"); + var summaryUriSecondPage = new Uri("https://access.redhat.com/hydra/rest/securitydata/csaf.json?after=2025-10-03&per_page=10&page=2"); + var summaryUriRepeatOverlap = new Uri("https://access.redhat.com/hydra/rest/securitydata/csaf.json?after=2025-10-04&per_page=10&page=1"); + var summaryUriSecondPageOverlap = new Uri("https://access.redhat.com/hydra/rest/securitydata/csaf.json?after=2025-10-04&per_page=10&page=2"); + + _output.WriteLine($"Registering repeat summary URI: {summaryUriRepeat}"); + _output.WriteLine($"Registering second page summary URI: {summaryUriSecondPage}"); + _output.WriteLine($"Registering overlap repeat summary URI: {summaryUriRepeatOverlap}"); + _output.WriteLine($"Registering overlap second page summary URI: {summaryUriSecondPageOverlap}"); + _handler.AddJsonResponse(summaryUriRepeat, ReadFixture("summary-page1-repeat.json")); + _handler.AddJsonResponse(summaryUriSecondPage, ReadFixture("summary-page2.json")); + _handler.AddJsonResponse(summaryUriRepeatOverlap, ReadFixture("summary-page1-repeat.json")); + _handler.AddJsonResponse(summaryUriSecondPageOverlap, ReadFixture("summary-page2.json")); + + await connector.FetchAsync(provider, CancellationToken.None); + await connector.ParseAsync(provider, CancellationToken.None); + await connector.MapAsync(provider, CancellationToken.None); + + advisories = await advisoryStore.GetRecentAsync(10, CancellationToken.None); + Assert.Equal(2, advisories.Count); + + var secondAdvisory = advisories.Single(a => string.Equals(a.AdvisoryKey, "RHSA-2025:0002", StringComparison.Ordinal)); + var rpm2 = secondAdvisory.AffectedPackages.Single(pkg => pkg.Type == AffectedPackageTypes.Rpm); + Assert.Equal("kernel-0:5.14.0-400.el9.x86_64", rpm2.Identifier); + const string knownNotAffected = "known_not_affected"; + + foreach (var status in rpm2.Statuses) + { + _output.WriteLine($"RPM2 status: {status.Status}"); + } + + Assert.DoesNotContain(rpm2.VersionRanges, range => string.Equals(range.RangeExpression, knownNotAffected, StringComparison.Ordinal)); + Assert.Contains(rpm2.Statuses, status => status.Status == knownNotAffected); + + var cpe2 = secondAdvisory.AffectedPackages.Single(pkg => pkg.Type == AffectedPackageTypes.Cpe); + Assert.Equal("cpe:2.3:o:redhat:enterprise_linux:9:*:*:*:*:*:*:*", cpe2.Identifier); + Assert.Empty(cpe2.VersionRanges); + Assert.Contains(cpe2.Statuses, status => status.Status == knownNotAffected); + + state = await stateRepository.TryGetAsync(RedHatConnectorPlugin.SourceName, CancellationToken.None); + Assert.NotNull(state); + Assert.True(state!.Cursor.TryGetValue("pendingDocuments", out var pendingDocs3) && pendingDocs3.AsBsonArray.Count == 0); + Assert.True(state.Cursor.TryGetValue("pendingMappings", out var pendingMappings3) && pendingMappings3.AsBsonArray.Count == 0); + } + + [Fact] + public void GoldenFixturesMatchSnapshots() + { + var fixtures = new[] + { + new GoldenFixtureCase( + AdvisoryId: "RHSA-2025:0001", + InputFile: "csaf-rhsa-2025-0001.json", + SnapshotFile: "rhsa-2025-0001.snapshot.json", + ValidatedAt: DateTimeOffset.Parse("2025-10-05T00:00:00Z")), + new GoldenFixtureCase( + AdvisoryId: "RHSA-2025:0002", + InputFile: "csaf-rhsa-2025-0002.json", + SnapshotFile: "rhsa-2025-0002.snapshot.json", + ValidatedAt: DateTimeOffset.Parse("2025-10-05T12:00:00Z")), + new GoldenFixtureCase( + AdvisoryId: "RHSA-2025:0003", + InputFile: "csaf-rhsa-2025-0003.json", + SnapshotFile: "rhsa-2025-0003.snapshot.json", + ValidatedAt: DateTimeOffset.Parse("2025-10-06T09:00:00Z")), + }; + + var updateGoldens = ShouldUpdateGoldens(); + + foreach (var fixture in fixtures) + { + var snapshot = MapFixtureToSnapshot(fixture); + var snapshotPath = ProjectFixturePath(fixture.SnapshotFile); + + if (updateGoldens) + { + File.WriteAllText(snapshotPath, snapshot); + continue; + } + + var expected = File.ReadAllText(snapshotPath).Replace("\r\n", "\n"); + Assert.Equal(expected, snapshot); + } + } + + [Fact] + public async Task Resume_CompletesPendingDocumentsAfterRestart() + { + await ResetDatabaseAsync(); + + var options = new RedHatOptions + { + BaseEndpoint = new Uri("https://access.redhat.com/hydra/rest/securitydata"), + PageSize = 10, + MaxPagesPerFetch = 2, + MaxAdvisoriesPerFetch = 25, + InitialBackfill = TimeSpan.FromDays(1), + Overlap = TimeSpan.Zero, + FetchTimeout = TimeSpan.FromSeconds(30), + UserAgent = "StellaOps.Tests.RedHat/1.0", + }; + + var summaryUri = new Uri("https://access.redhat.com/hydra/rest/securitydata/csaf.json?after=2025-10-04&per_page=10&page=1"); + var summaryUriPost = new Uri("https://access.redhat.com/hydra/rest/securitydata/csaf.json?after=2025-10-05&per_page=10&page=1"); + var summaryUriPostPage2 = new Uri("https://access.redhat.com/hydra/rest/securitydata/csaf.json?after=2025-10-05&per_page=10&page=2"); + var detailUri = new Uri("https://access.redhat.com/hydra/rest/securitydata/csaf/RHSA-2025:0001.json"); + var detailUri2 = new Uri("https://access.redhat.com/hydra/rest/securitydata/csaf/RHSA-2025:0002.json"); + + var fetchHandler = new CannedHttpMessageHandler(); + fetchHandler.AddJsonResponse(summaryUri, ReadFixture("summary-page1-repeat.json")); + fetchHandler.AddJsonResponse(summaryUriPost, "[]"); + fetchHandler.AddJsonResponse(summaryUriPostPage2, "[]"); + fetchHandler.AddJsonResponse(detailUri, ReadFixture("csaf-rhsa-2025-0001.json")); + fetchHandler.AddJsonResponse(detailUri2, ReadFixture("csaf-rhsa-2025-0002.json")); + + Guid[] pendingDocumentIds; + await using (var fetchProvider = await CreateServiceProviderAsync(options, fetchHandler)) + { + var stateRepository = fetchProvider.GetRequiredService<ISourceStateRepository>(); + await stateRepository.UpsertAsync( + new SourceStateRecord( + RedHatConnectorPlugin.SourceName, + Enabled: true, + Paused: false, + Cursor: new BsonDocument(), + LastSuccess: null, + LastFailure: null, + FailCount: 0, + BackoffUntil: null, + UpdatedAt: _timeProvider.GetUtcNow(), + LastFailureReason: null), + CancellationToken.None); + + var connector = new RedHatConnectorPlugin().Create(fetchProvider); + await connector.FetchAsync(fetchProvider, CancellationToken.None); + + var state = await stateRepository.TryGetAsync(RedHatConnectorPlugin.SourceName, CancellationToken.None); + Assert.NotNull(state); + var pendingDocs = state!.Cursor.TryGetValue("pendingDocuments", out var pendingDocsValue) + ? pendingDocsValue.AsBsonArray + : new BsonArray(); + Assert.NotEmpty(pendingDocs); + pendingDocumentIds = pendingDocs.Select(value => Guid.Parse(value.AsString)).ToArray(); + } + + var resumeHandler = new CannedHttpMessageHandler(); + await using (var resumeProvider = await CreateServiceProviderAsync(options, resumeHandler)) + { + var resumeConnector = new RedHatConnectorPlugin().Create(resumeProvider); + + await resumeConnector.ParseAsync(resumeProvider, CancellationToken.None); + await resumeConnector.MapAsync(resumeProvider, CancellationToken.None); + + var documentStore = resumeProvider.GetRequiredService<IDocumentStore>(); + foreach (var documentId in pendingDocumentIds) + { + var document = await documentStore.FindAsync(documentId, CancellationToken.None); + Assert.NotNull(document); + Assert.Equal(DocumentStatuses.Mapped, document!.Status); + } + + var advisoryStore = resumeProvider.GetRequiredService<IAdvisoryStore>(); + var advisories = await advisoryStore.GetRecentAsync(10, CancellationToken.None); + Assert.NotEmpty(advisories); + + var stateRepository = resumeProvider.GetRequiredService<ISourceStateRepository>(); + var finalState = await stateRepository.TryGetAsync(RedHatConnectorPlugin.SourceName, CancellationToken.None); + Assert.NotNull(finalState); + var finalPendingDocs = finalState!.Cursor.TryGetValue("pendingDocuments", out var docsValue) ? docsValue.AsBsonArray : new BsonArray(); + Assert.Empty(finalPendingDocs); + var finalPendingMappings = finalState.Cursor.TryGetValue("pendingMappings", out var mappingsValue) ? mappingsValue.AsBsonArray : new BsonArray(); + Assert.Empty(finalPendingMappings); + } + } + + [Fact] + public async Task MapAsync_DeduplicatesReferencesAndOrdersDeterministically() + { + await ResetDatabaseAsync(); + + var options = new RedHatOptions + { + BaseEndpoint = new Uri("https://access.redhat.com/hydra/rest/securitydata"), + PageSize = 10, + MaxPagesPerFetch = 2, + MaxAdvisoriesPerFetch = 10, + InitialBackfill = TimeSpan.FromDays(7), + Overlap = TimeSpan.Zero, + FetchTimeout = TimeSpan.FromSeconds(30), + UserAgent = "StellaOps.Tests.RedHat/1.0", + }; + + await EnsureServiceProviderAsync(options); + var provider = _serviceProvider!; + + var summaryUri = new Uri("https://access.redhat.com/hydra/rest/securitydata/csaf.json?after=2025-09-28&per_page=10&page=1"); + var summaryUriPost = new Uri("https://access.redhat.com/hydra/rest/securitydata/csaf.json?after=2025-10-05&per_page=10&page=1"); + var detailUri = new Uri("https://access.redhat.com/hydra/rest/securitydata/csaf/RHSA-2025:0003.json"); + + _handler.AddJsonResponse(summaryUri, ReadFixture("summary-page3.json")); + _handler.AddJsonResponse(summaryUriPost, "[]"); + _handler.AddJsonResponse(detailUri, ReadFixture("csaf-rhsa-2025-0003.json")); + + var stateRepository = provider.GetRequiredService<ISourceStateRepository>(); + await stateRepository.UpsertAsync( + new SourceStateRecord( + RedHatConnectorPlugin.SourceName, + Enabled: true, + Paused: false, + Cursor: new BsonDocument(), + LastSuccess: null, + LastFailure: null, + FailCount: 0, + BackoffUntil: null, + UpdatedAt: _timeProvider.GetUtcNow(), + LastFailureReason: null), + CancellationToken.None); + + var connector = new RedHatConnectorPlugin().Create(provider); + + await connector.FetchAsync(provider, CancellationToken.None); + await connector.ParseAsync(provider, CancellationToken.None); + await connector.MapAsync(provider, CancellationToken.None); + + var advisoryStore = provider.GetRequiredService<IAdvisoryStore>(); + var advisory = (await advisoryStore.GetRecentAsync(10, CancellationToken.None)) + .Single(a => string.Equals(a.AdvisoryKey, "RHSA-2025:0003", StringComparison.Ordinal)); + + var references = advisory.References.ToArray(); + Assert.Collection( + references, + reference => + { + Assert.Equal("self", reference.Kind); + Assert.Equal("https://access.redhat.com/errata/RHSA-2025:0003", reference.Url); + Assert.Equal("Primary advisory", reference.Summary); + }, + reference => + { + Assert.Equal("mitigation", reference.Kind); + Assert.Equal("https://access.redhat.com/solutions/999999", reference.Url); + Assert.Equal("Knowledge base guidance", reference.Summary); + }, + reference => + { + Assert.Equal("exploit", reference.Kind); + Assert.Equal("https://bugzilla.redhat.com/show_bug.cgi?id=2222222", reference.Url); + Assert.Equal("Exploit tracking", reference.Summary); + }, + reference => + { + Assert.Equal("external", reference.Kind); + Assert.Equal("https://www.cve.org/CVERecord?id=CVE-2025-0003", reference.Url); + Assert.Equal("CVE record", reference.Summary); + }); + Assert.Equal(4, references.Length); + + Assert.Equal("self", references[0].Kind); + Assert.Equal("https://access.redhat.com/errata/RHSA-2025:0003", references[0].Url); + Assert.Equal("Primary advisory", references[0].Summary); + + Assert.Equal("mitigation", references[1].Kind); + Assert.Equal("https://access.redhat.com/solutions/999999", references[1].Url); + Assert.Equal("Knowledge base guidance", references[1].Summary); + + Assert.Equal("exploit", references[2].Kind); + Assert.Equal("https://bugzilla.redhat.com/show_bug.cgi?id=2222222", references[2].Url); + + Assert.Equal("external", references[3].Kind); + Assert.Equal("https://www.cve.org/CVERecord?id=CVE-2025-0003", references[3].Url); + Assert.Equal("CVE record", references[3].Summary); + } + + private static string MapFixtureToSnapshot(GoldenFixtureCase fixture) + { + var jsonPath = ProjectFixturePath(fixture.InputFile); + var json = File.ReadAllText(jsonPath); + + using var jsonDocument = JsonDocument.Parse(json); + var bson = BsonDocument.Parse(json); + + var metadata = new Dictionary<string, string>(StringComparer.OrdinalIgnoreCase) + { + ["advisoryId"] = fixture.AdvisoryId, + }; + + var document = new DocumentRecord( + Guid.NewGuid(), + RedHatConnectorPlugin.SourceName, + $"https://access.redhat.com/hydra/rest/securitydata/csaf/{fixture.AdvisoryId}.json", + fixture.ValidatedAt, + new string('0', 64), + DocumentStatuses.Mapped, + "application/json", + Headers: null, + Metadata: metadata, + Etag: null, + LastModified: fixture.ValidatedAt, + GridFsId: null); + + var dto = new DtoRecord(Guid.NewGuid(), document.Id, RedHatConnectorPlugin.SourceName, "redhat.csaf.v2", bson, fixture.ValidatedAt); + + var advisory = RedHatMapper.Map(RedHatConnectorPlugin.SourceName, dto, document, jsonDocument); + Assert.NotNull(advisory); + + return SnapshotSerializer.ToSnapshot(advisory!).Replace("\r\n", "\n"); + } + + private static bool ShouldUpdateGoldens() + => ForceUpdateGoldens + || IsTruthy(Environment.GetEnvironmentVariable("UPDATE_GOLDENS")) + || IsTruthy(Environment.GetEnvironmentVariable("DOTNET_TEST_UPDATE_GOLDENS")); + + private static bool IsTruthy(string? value) + => !string.IsNullOrWhiteSpace(value) + && (string.Equals(value, "1", StringComparison.OrdinalIgnoreCase) + || string.Equals(value, "true", StringComparison.OrdinalIgnoreCase) + || string.Equals(value, "yes", StringComparison.OrdinalIgnoreCase)); + + private sealed record GoldenFixtureCase(string AdvisoryId, string InputFile, string SnapshotFile, DateTimeOffset ValidatedAt); + + private static string ProjectFixturePath(string filename) + => Path.Combine(GetProjectRoot(), "RedHat", "Fixtures", filename); + + private static string GetProjectRoot() + => Path.GetFullPath(Path.Combine(AppContext.BaseDirectory, "..", "..", "..")); + + private async Task EnsureServiceProviderAsync(RedHatOptions options) + { + if (_serviceProvider is not null) + { + return; + } + + _serviceProvider = await CreateServiceProviderAsync(options, _handler); + } + + private async Task<ServiceProvider> CreateServiceProviderAsync(RedHatOptions options, CannedHttpMessageHandler handler) + { + var services = new ServiceCollection(); + services.AddLogging(builder => builder.AddProvider(NullLoggerProvider.Instance)); + services.AddSingleton<TimeProvider>(_timeProvider); + services.AddSingleton(handler); + + services.AddMongoStorage(storageOptions => + { + storageOptions.ConnectionString = _fixture.Runner.ConnectionString; + storageOptions.DatabaseName = _fixture.Database.DatabaseNamespace.DatabaseName; + storageOptions.CommandTimeout = TimeSpan.FromSeconds(5); + }); + + services.AddSourceCommon(); + services.AddRedHatConnector(opts => + { + opts.BaseEndpoint = options.BaseEndpoint; + opts.SummaryPath = options.SummaryPath; + opts.PageSize = options.PageSize; + opts.MaxPagesPerFetch = options.MaxPagesPerFetch; + opts.MaxAdvisoriesPerFetch = options.MaxAdvisoriesPerFetch; + opts.InitialBackfill = options.InitialBackfill; + opts.Overlap = options.Overlap; + opts.FetchTimeout = options.FetchTimeout; + opts.UserAgent = options.UserAgent; + }); + + services.Configure<JobSchedulerOptions>(schedulerOptions => + { + var fetchType = Type.GetType("StellaOps.Feedser.Source.Distro.RedHat.RedHatFetchJob, StellaOps.Feedser.Source.Distro.RedHat", throwOnError: true)!; + var parseType = Type.GetType("StellaOps.Feedser.Source.Distro.RedHat.RedHatParseJob, StellaOps.Feedser.Source.Distro.RedHat", throwOnError: true)!; + var mapType = Type.GetType("StellaOps.Feedser.Source.Distro.RedHat.RedHatMapJob, StellaOps.Feedser.Source.Distro.RedHat", throwOnError: true)!; + + schedulerOptions.Definitions["source:redhat:fetch"] = new JobDefinition("source:redhat:fetch", fetchType, TimeSpan.FromMinutes(12), TimeSpan.FromMinutes(6), "0,15,30,45 * * * *", true); + schedulerOptions.Definitions["source:redhat:parse"] = new JobDefinition("source:redhat:parse", parseType, TimeSpan.FromMinutes(15), TimeSpan.FromMinutes(6), "5,20,35,50 * * * *", true); + schedulerOptions.Definitions["source:redhat:map"] = new JobDefinition("source:redhat:map", mapType, TimeSpan.FromMinutes(20), TimeSpan.FromMinutes(6), "10,25,40,55 * * * *", true); + }); + + services.Configure<HttpClientFactoryOptions>(RedHatOptions.HttpClientName, builderOptions => + { + builderOptions.HttpMessageHandlerBuilderActions.Add(builder => + { + builder.PrimaryHandler = handler; + }); + }); + + var provider = services.BuildServiceProvider(); + var bootstrapper = provider.GetRequiredService<MongoBootstrapper>(); + await bootstrapper.InitializeAsync(CancellationToken.None); + return provider; + } + + private Task ResetDatabaseAsync() + { + return ResetDatabaseInternalAsync(); + } + + private async Task ResetDatabaseInternalAsync() + { + if (_serviceProvider is not null) + { + if (_serviceProvider is IAsyncDisposable asyncDisposable) + { + await asyncDisposable.DisposeAsync(); + } + else + { + _serviceProvider.Dispose(); + } + + _serviceProvider = null; + } + + await _fixture.Client.DropDatabaseAsync(_fixture.Database.DatabaseNamespace.DatabaseName); + _handler.Clear(); + _timeProvider.SetUtcNow(_initialNow); + } + + private static string ReadFixture(string name) + => File.ReadAllText(ResolveFixturePath(name)); + + private static string ResolveFixturePath(string filename) + { + var candidates = new[] + { + Path.Combine(AppContext.BaseDirectory, "Source", "Distro", "RedHat", "Fixtures", filename), + Path.Combine(AppContext.BaseDirectory, "RedHat", "Fixtures", filename), + }; + + foreach (var candidate in candidates) + { + if (File.Exists(candidate)) + { + return candidate; + } + } + + throw new FileNotFoundException($"Fixture '{filename}' not found in output directory.", filename); + } + + private static string NormalizeLineEndings(string value) + { + var normalized = value.Replace("\r\n", "\n").Replace('\r', '\n'); + return normalized.TrimEnd('\n'); + } + + public Task InitializeAsync() => Task.CompletedTask; + + public async Task DisposeAsync() + { + await ResetDatabaseInternalAsync(); + } +} diff --git a/src/StellaOps.Feedser.Source.Distro.RedHat.Tests/StellaOps.Feedser.Source.Distro.RedHat.Tests.csproj b/src/StellaOps.Feedser.Source.Distro.RedHat.Tests/StellaOps.Feedser.Source.Distro.RedHat.Tests.csproj new file mode 100644 index 00000000..654d8c5f --- /dev/null +++ b/src/StellaOps.Feedser.Source.Distro.RedHat.Tests/StellaOps.Feedser.Source.Distro.RedHat.Tests.csproj @@ -0,0 +1,18 @@ +<Project Sdk="Microsoft.NET.Sdk"> + <PropertyGroup> + <TargetFramework>net10.0</TargetFramework> + <ImplicitUsings>enable</ImplicitUsings> + <Nullable>enable</Nullable> + </PropertyGroup> + <ItemGroup> + <ProjectReference Include="../StellaOps.Feedser.Models/StellaOps.Feedser.Models.csproj" /> + <ProjectReference Include="../StellaOps.Feedser.Source.Common/StellaOps.Feedser.Source.Common.csproj" /> + <ProjectReference Include="../StellaOps.Feedser.Source.Distro.RedHat/StellaOps.Feedser.Source.Distro.RedHat.csproj" /> + <ProjectReference Include="../StellaOps.Feedser.Storage.Mongo/StellaOps.Feedser.Storage.Mongo.csproj" /> + </ItemGroup> + <ItemGroup> + <None Include="RedHat/Fixtures/*.json" + CopyToOutputDirectory="Always" + TargetPath="Source/Distro/RedHat/Fixtures/%(Filename)%(Extension)" /> + </ItemGroup> +</Project> diff --git a/src/StellaOps.Feedser.Source.Distro.RedHat/AGENTS.md b/src/StellaOps.Feedser.Source.Distro.RedHat/AGENTS.md new file mode 100644 index 00000000..89956ea5 --- /dev/null +++ b/src/StellaOps.Feedser.Source.Distro.RedHat/AGENTS.md @@ -0,0 +1,27 @@ +# AGENTS +## Role +Red Hat distro connector (Security Data API and OVAL) providing authoritative OS package ranges (RPM NEVRA) and RHSA metadata; overrides generic registry ranges during merge. +## Scope +- Fetch Security Data JSON (for example CVRF) via Hydra; window by last_modified or after cursor; optionally ingest OVAL definitions. +- Validate payloads; parse advisories, CVEs, affected packages; materialize NEVRA and CPE records. +- Map to canonical advisories with affected Type=rpm/cpe, fixedBy NEVRA, RHSA aliasing; persist provenance indicating oval/package.nevra. +## Participants +- Source.Common (HTTP, throttling, validators). +- Storage.Mongo (document, dto, advisory, alias, affected, reference, source_state). +- Models (canonical Affected with NEVRA). +- Core/WebService (jobs: source:redhat:fetch|parse|map) already registered. +- Merge engine to enforce distro precedence (OVAL or PSIRT greater than NVD). +## Interfaces & contracts +- Aliases: RHSA-YYYY:NNNN, CVE ids; references include RHSA pages, errata, OVAL links. +- Affected: rpm (Identifier=NEVRA key) and cpe entries; versions include introduced/fixed/fixedBy; platforms mark RHEL streams. +- Provenance: kind="oval" or "package.nevra" as applicable; value=definition id or package. +## In/Out of scope +In: authoritative rpm ranges, RHSA mapping, OVAL interpretation, watermarking. +Out: building RPM artifacts; cross-distro reconciliation beyond Red Hat. +## Observability & security expectations +- Metrics: SourceDiagnostics publishes `feedser.source.http.*` counters/histograms tagged `feedser.source=redhat`, capturing fetch volumes, parse/OVAL failures, and map affected counts without bespoke metric names. +- Logs: cursor bounds, advisory ids, NEVRA counts; allowlist Red Hat endpoints. +## Tests +- Author and review coverage in `../StellaOps.Feedser.Source.Distro.RedHat.Tests`. +- Shared fixtures (e.g., `MongoIntegrationFixture`, `ConnectorTestHarness`) live in `../StellaOps.Feedser.Testing`. +- Keep fixtures deterministic; match new cases to real-world advisories or regression scenarios. diff --git a/src/StellaOps.Feedser.Source.Distro.RedHat/CONFLICT_RESOLVER_NOTES.md b/src/StellaOps.Feedser.Source.Distro.RedHat/CONFLICT_RESOLVER_NOTES.md new file mode 100644 index 00000000..55298baf --- /dev/null +++ b/src/StellaOps.Feedser.Source.Distro.RedHat/CONFLICT_RESOLVER_NOTES.md @@ -0,0 +1,25 @@ +# RHSA Fixture Diffs for Conflict Resolver (Sprint 1) + +_Status date: 2025-10-11_ + +The Red Hat connector fixtures were re-baselined after the model helper rollout so that the conflict resolver receives the canonical payload shape expected for range reconciliation. + +## Key schema deltas + +- `affectedPackages[]` now emits the `type` field ahead of the identifier and always carries a `normalizedVersions` array (empty for NEVRA/CPE today) alongside existing `versionRanges`. +- All nested `provenance` objects (package ranges, statuses, advisory-level metadata, references) now serialize in canonical order – `source`, `kind`, `value`, `decisionReason`, `recordedAt`, `fieldMask` – to align with `AdvisoryProvenance` equality used by the conflict resolver. +- `decisionReason` is now present (null) on provenance payloads so future precedence decisions can annotate overrides without another fixture bump. + +## Impact on conflict resolver + +- Range merge logic must accept an optional `normalizedVersions` array even when it is empty; RPM reconciliation continues to rely on NEVRA primitives (`rangeKind: "nevra"`). +- Provenance comparisons should treat the new property ordering and `decisionReason` field as canonical; older snapshots that lacked these fields are obsolete. +- Advisory/reference provenance now matches the structure that merge emits, so deterministic hashing of resolver inputs will remain stable across connectors. + +## Updated goldens + +- `src/StellaOps.Feedser.Source.Distro.RedHat.Tests/RedHat/Fixtures/rhsa-2025-0001.snapshot.json` +- `src/StellaOps.Feedser.Source.Distro.RedHat.Tests/RedHat/Fixtures/rhsa-2025-0002.snapshot.json` +- `src/StellaOps.Feedser.Source.Distro.RedHat.Tests/RedHat/Fixtures/rhsa-2025-0003.snapshot.json` + +Keep these notes in sync with any future provenance or normalized-rule updates so the conflict resolver team can reason about fixture-driven regressions. diff --git a/src/StellaOps.Feedser.Source.Distro.RedHat/Configuration/RedHatOptions.cs b/src/StellaOps.Feedser.Source.Distro.RedHat/Configuration/RedHatOptions.cs new file mode 100644 index 00000000..81739d7a --- /dev/null +++ b/src/StellaOps.Feedser.Source.Distro.RedHat/Configuration/RedHatOptions.cs @@ -0,0 +1,97 @@ +namespace StellaOps.Feedser.Source.Distro.RedHat.Configuration; + +public sealed class RedHatOptions +{ + /// <summary> + /// Name of the HttpClient registered for Red Hat Hydra requests. + /// </summary> + public const string HttpClientName = "redhat-hydra"; + + /// <summary> + /// Base API endpoint for Hydra security data requests. + /// </summary> + public Uri BaseEndpoint { get; set; } = new("https://access.redhat.com/hydra/rest/securitydata"); + + /// <summary> + /// Relative path for the advisory listing endpoint (returns summary rows with resource_url values). + /// </summary> + public string SummaryPath { get; set; } = "csaf.json"; + + /// <summary> + /// Number of summary rows requested per page when scanning for new advisories. + /// </summary> + public int PageSize { get; set; } = 200; + + /// <summary> + /// Maximum number of summary pages to inspect within one fetch invocation. + /// </summary> + public int MaxPagesPerFetch { get; set; } = 5; + + /// <summary> + /// Upper bound on individual advisories fetched per invocation (guards against unbounded catch-up floods). + /// </summary> + public int MaxAdvisoriesPerFetch { get; set; } = 800; + + /// <summary> + /// Initial look-back window applied when no watermark exists (Red Hat publishes extensive history; we default to 30 days). + /// </summary> + public TimeSpan InitialBackfill { get; set; } = TimeSpan.FromDays(30); + + /// <summary> + /// Optional overlap period re-scanned on each run to pick up late-published advisories. + /// </summary> + public TimeSpan Overlap { get; set; } = TimeSpan.FromDays(1); + + /// <summary> + /// Timeout applied to individual Hydra document fetches. + /// </summary> + public TimeSpan FetchTimeout { get; set; } = TimeSpan.FromSeconds(60); + + /// <summary> + /// Custom user-agent presented to Red Hat endpoints (kept short to satisfy Jetty header limits). + /// </summary> + public string UserAgent { get; set; } = "StellaOps.Feedser.RedHat/1.0"; + + public void Validate() + { + if (BaseEndpoint is null || !BaseEndpoint.IsAbsoluteUri) + { + throw new InvalidOperationException("Red Hat Hydra base endpoint must be an absolute URI."); + } + + if (string.IsNullOrWhiteSpace(SummaryPath)) + { + throw new InvalidOperationException("Red Hat Hydra summary path must be configured."); + } + + if (PageSize <= 0) + { + throw new InvalidOperationException("Red Hat Hydra page size must be positive."); + } + + if (MaxPagesPerFetch <= 0) + { + throw new InvalidOperationException("Red Hat Hydra max pages per fetch must be positive."); + } + + if (MaxAdvisoriesPerFetch <= 0) + { + throw new InvalidOperationException("Red Hat Hydra max advisories per fetch must be positive."); + } + + if (InitialBackfill <= TimeSpan.Zero) + { + throw new InvalidOperationException("Red Hat Hydra initial backfill must be positive."); + } + + if (Overlap < TimeSpan.Zero) + { + throw new InvalidOperationException("Red Hat Hydra overlap cannot be negative."); + } + + if (FetchTimeout <= TimeSpan.Zero) + { + throw new InvalidOperationException("Red Hat Hydra fetch timeout must be positive."); + } + } +} diff --git a/src/StellaOps.Feedser.Source.Distro.RedHat/Internal/Models/RedHatCsafModels.cs b/src/StellaOps.Feedser.Source.Distro.RedHat/Internal/Models/RedHatCsafModels.cs new file mode 100644 index 00000000..3fc6375e --- /dev/null +++ b/src/StellaOps.Feedser.Source.Distro.RedHat/Internal/Models/RedHatCsafModels.cs @@ -0,0 +1,177 @@ +using System; +using System.Collections.Generic; +using System.Text.Json.Serialization; + +namespace StellaOps.Feedser.Source.Distro.RedHat.Internal.Models; + +internal sealed class RedHatCsafEnvelope +{ + [JsonPropertyName("document")] + public RedHatDocumentSection? Document { get; init; } + + [JsonPropertyName("product_tree")] + public RedHatProductTree? ProductTree { get; init; } + + [JsonPropertyName("vulnerabilities")] + public IReadOnlyList<RedHatVulnerability>? Vulnerabilities { get; init; } +} + +internal sealed class RedHatDocumentSection +{ + [JsonPropertyName("aggregate_severity")] + public RedHatAggregateSeverity? AggregateSeverity { get; init; } + + [JsonPropertyName("lang")] + public string? Lang { get; init; } + + [JsonPropertyName("notes")] + public IReadOnlyList<RedHatDocumentNote>? Notes { get; init; } + + [JsonPropertyName("references")] + public IReadOnlyList<RedHatReference>? References { get; init; } + + [JsonPropertyName("title")] + public string? Title { get; init; } + + [JsonPropertyName("tracking")] + public RedHatTracking? Tracking { get; init; } +} + +internal sealed class RedHatAggregateSeverity +{ + [JsonPropertyName("text")] + public string? Text { get; init; } +} + +internal sealed class RedHatDocumentNote +{ + [JsonPropertyName("category")] + public string? Category { get; init; } + + [JsonPropertyName("text")] + public string? Text { get; init; } + + public bool CategoryEquals(string value) + => !string.IsNullOrWhiteSpace(Category) + && string.Equals(Category, value, StringComparison.OrdinalIgnoreCase); +} + +internal sealed class RedHatTracking +{ + [JsonPropertyName("id")] + public string? Id { get; init; } + + [JsonPropertyName("initial_release_date")] + public DateTimeOffset? InitialReleaseDate { get; init; } + + [JsonPropertyName("current_release_date")] + public DateTimeOffset? CurrentReleaseDate { get; init; } +} + +internal sealed class RedHatReference +{ + [JsonPropertyName("category")] + public string? Category { get; init; } + + [JsonPropertyName("summary")] + public string? Summary { get; init; } + + [JsonPropertyName("url")] + public string? Url { get; init; } +} + +internal sealed class RedHatProductTree +{ + [JsonPropertyName("branches")] + public IReadOnlyList<RedHatProductBranch>? Branches { get; init; } +} + +internal sealed class RedHatProductBranch +{ + [JsonPropertyName("category")] + public string? Category { get; init; } + + [JsonPropertyName("name")] + public string? Name { get; init; } + + [JsonPropertyName("product")] + public RedHatProductNodeInfo? Product { get; init; } + + [JsonPropertyName("branches")] + public IReadOnlyList<RedHatProductBranch>? Branches { get; init; } +} + +internal sealed class RedHatProductNodeInfo +{ + [JsonPropertyName("name")] + public string? Name { get; init; } + + [JsonPropertyName("product_id")] + public string? ProductId { get; init; } + + [JsonPropertyName("product_identification_helper")] + public RedHatProductIdentificationHelper? ProductIdentificationHelper { get; init; } +} + +internal sealed class RedHatProductIdentificationHelper +{ + [JsonPropertyName("cpe")] + public string? Cpe { get; init; } + + [JsonPropertyName("purl")] + public string? Purl { get; init; } +} + +internal sealed class RedHatVulnerability +{ + [JsonPropertyName("cve")] + public string? Cve { get; init; } + + [JsonPropertyName("references")] + public IReadOnlyList<RedHatReference>? References { get; init; } + + [JsonPropertyName("scores")] + public IReadOnlyList<RedHatVulnerabilityScore>? Scores { get; init; } + + [JsonPropertyName("product_status")] + public RedHatProductStatus? ProductStatus { get; init; } +} + +internal sealed class RedHatVulnerabilityScore +{ + [JsonPropertyName("cvss_v3")] + public RedHatCvssV3? CvssV3 { get; init; } +} + +internal sealed class RedHatCvssV3 +{ + [JsonPropertyName("baseScore")] + public double? BaseScore { get; init; } + + [JsonPropertyName("baseSeverity")] + public string? BaseSeverity { get; init; } + + [JsonPropertyName("vectorString")] + public string? VectorString { get; init; } + + [JsonPropertyName("version")] + public string? Version { get; init; } +} + +internal sealed class RedHatProductStatus +{ + [JsonPropertyName("fixed")] + public IReadOnlyList<string>? Fixed { get; init; } + + [JsonPropertyName("first_fixed")] + public IReadOnlyList<string>? FirstFixed { get; init; } + + [JsonPropertyName("known_affected")] + public IReadOnlyList<string>? KnownAffected { get; init; } + + [JsonPropertyName("known_not_affected")] + public IReadOnlyList<string>? KnownNotAffected { get; init; } + + [JsonPropertyName("under_investigation")] + public IReadOnlyList<string>? UnderInvestigation { get; init; } +} diff --git a/src/StellaOps.Feedser.Source.Distro.RedHat/Internal/RedHatCursor.cs b/src/StellaOps.Feedser.Source.Distro.RedHat/Internal/RedHatCursor.cs new file mode 100644 index 00000000..cbb807d1 --- /dev/null +++ b/src/StellaOps.Feedser.Source.Distro.RedHat/Internal/RedHatCursor.cs @@ -0,0 +1,254 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using MongoDB.Bson; + +namespace StellaOps.Feedser.Source.Distro.RedHat.Internal; + +internal sealed record RedHatCursor( + DateTimeOffset? LastReleasedOn, + IReadOnlyCollection<string> ProcessedAdvisoryIds, + IReadOnlyCollection<Guid> PendingDocuments, + IReadOnlyCollection<Guid> PendingMappings, + IReadOnlyDictionary<string, RedHatCachedFetchMetadata> FetchCache) +{ + private static readonly IReadOnlyCollection<string> EmptyStringList = Array.Empty<string>(); + private static readonly IReadOnlyCollection<Guid> EmptyGuidList = Array.Empty<Guid>(); + private static readonly IReadOnlyDictionary<string, RedHatCachedFetchMetadata> EmptyCache = + new Dictionary<string, RedHatCachedFetchMetadata>(StringComparer.OrdinalIgnoreCase); + + public static RedHatCursor Empty { get; } = new(null, EmptyStringList, EmptyGuidList, EmptyGuidList, EmptyCache); + + public static RedHatCursor FromBsonDocument(BsonDocument? document) + { + if (document is null || document.ElementCount == 0) + { + return Empty; + } + + DateTimeOffset? lastReleased = null; + if (document.TryGetValue("lastReleasedOn", out var lastReleasedValue)) + { + lastReleased = ReadDateTimeOffset(lastReleasedValue); + } + + var processed = ReadStringSet(document, "processedAdvisories"); + var pendingDocuments = ReadGuidSet(document, "pendingDocuments"); + var pendingMappings = ReadGuidSet(document, "pendingMappings"); + var fetchCache = ReadFetchCache(document); + + return new RedHatCursor(lastReleased, processed, pendingDocuments, pendingMappings, fetchCache); + } + + public BsonDocument ToBsonDocument() + { + var document = new BsonDocument(); + if (LastReleasedOn.HasValue) + { + document["lastReleasedOn"] = LastReleasedOn.Value.UtcDateTime; + } + + document["processedAdvisories"] = new BsonArray(ProcessedAdvisoryIds); + document["pendingDocuments"] = new BsonArray(PendingDocuments.Select(id => id.ToString())); + document["pendingMappings"] = new BsonArray(PendingMappings.Select(id => id.ToString())); + + var cacheArray = new BsonArray(); + foreach (var (key, metadata) in FetchCache) + { + var cacheDoc = new BsonDocument + { + ["uri"] = key + }; + + if (!string.IsNullOrWhiteSpace(metadata.ETag)) + { + cacheDoc["etag"] = metadata.ETag; + } + + if (metadata.LastModified.HasValue) + { + cacheDoc["lastModified"] = metadata.LastModified.Value.UtcDateTime; + } + + cacheArray.Add(cacheDoc); + } + + document["fetchCache"] = cacheArray; + return document; + } + + public RedHatCursor WithLastReleased(DateTimeOffset? releasedOn, IEnumerable<string> advisoryIds) + { + var normalizedIds = advisoryIds?.Where(static id => !string.IsNullOrWhiteSpace(id)) + .Select(static id => id.Trim()) + .Distinct(StringComparer.OrdinalIgnoreCase) + .ToArray() ?? Array.Empty<string>(); + + return this with + { + LastReleasedOn = releasedOn, + ProcessedAdvisoryIds = normalizedIds + }; + } + + public RedHatCursor AddProcessedAdvisories(IEnumerable<string> advisoryIds) + { + if (advisoryIds is null) + { + return this; + } + + var set = new HashSet<string>(ProcessedAdvisoryIds, StringComparer.OrdinalIgnoreCase); + foreach (var id in advisoryIds) + { + if (!string.IsNullOrWhiteSpace(id)) + { + set.Add(id.Trim()); + } + } + + return this with { ProcessedAdvisoryIds = set.ToArray() }; + } + + public RedHatCursor WithPendingDocuments(IEnumerable<Guid> ids) + { + var list = ids?.Distinct().ToArray() ?? Array.Empty<Guid>(); + return this with { PendingDocuments = list }; + } + + public RedHatCursor WithPendingMappings(IEnumerable<Guid> ids) + { + var list = ids?.Distinct().ToArray() ?? Array.Empty<Guid>(); + return this with { PendingMappings = list }; + } + + public RedHatCursor WithFetchCache(string requestUri, string? etag, DateTimeOffset? lastModified) + { + var cache = new Dictionary<string, RedHatCachedFetchMetadata>(FetchCache, StringComparer.OrdinalIgnoreCase) + { + [requestUri] = new RedHatCachedFetchMetadata(etag, lastModified) + }; + + return this with { FetchCache = cache }; + } + + public RedHatCursor PruneFetchCache(IEnumerable<string> keepUris) + { + if (FetchCache.Count == 0) + { + return this; + } + + var keepSet = new HashSet<string>(keepUris ?? Array.Empty<string>(), StringComparer.OrdinalIgnoreCase); + if (keepSet.Count == 0) + { + return this with { FetchCache = EmptyCache }; + } + + var cache = new Dictionary<string, RedHatCachedFetchMetadata>(StringComparer.OrdinalIgnoreCase); + foreach (var uri in keepSet) + { + if (FetchCache.TryGetValue(uri, out var metadata)) + { + cache[uri] = metadata; + } + } + + return this with { FetchCache = cache }; + } + + public RedHatCachedFetchMetadata? TryGetFetchCache(string requestUri) + { + if (FetchCache.TryGetValue(requestUri, out var metadata)) + { + return metadata; + } + + return null; + } + + private static IReadOnlyCollection<string> ReadStringSet(BsonDocument document, string field) + { + if (!document.TryGetValue(field, out var value) || value is not BsonArray array) + { + return EmptyStringList; + } + + var results = new List<string>(array.Count); + foreach (var element in array) + { + if (element.BsonType == BsonType.String) + { + var str = element.AsString.Trim(); + if (!string.IsNullOrWhiteSpace(str)) + { + results.Add(str); + } + } + } + + return results; + } + + private static IReadOnlyCollection<Guid> ReadGuidSet(BsonDocument document, string field) + { + if (!document.TryGetValue(field, out var value) || value is not BsonArray array) + { + return EmptyGuidList; + } + + var results = new List<Guid>(array.Count); + foreach (var element in array) + { + if (element.BsonType == BsonType.String && Guid.TryParse(element.AsString, out var guid)) + { + results.Add(guid); + } + } + + return results; + } + + private static IReadOnlyDictionary<string, RedHatCachedFetchMetadata> ReadFetchCache(BsonDocument document) + { + if (!document.TryGetValue("fetchCache", out var value) || value is not BsonArray array || array.Count == 0) + { + return EmptyCache; + } + + var results = new Dictionary<string, RedHatCachedFetchMetadata>(StringComparer.OrdinalIgnoreCase); + foreach (var element in array.OfType<BsonDocument>()) + { + if (!element.TryGetValue("uri", out var uriValue) || uriValue.BsonType != BsonType.String) + { + continue; + } + + var uri = uriValue.AsString; + var etag = element.TryGetValue("etag", out var etagValue) && etagValue.BsonType == BsonType.String + ? etagValue.AsString + : null; + DateTimeOffset? lastModified = null; + if (element.TryGetValue("lastModified", out var lastModifiedValue)) + { + lastModified = ReadDateTimeOffset(lastModifiedValue); + } + + results[uri] = new RedHatCachedFetchMetadata(etag, lastModified); + } + + return results; + } + + private static DateTimeOffset? ReadDateTimeOffset(BsonValue value) + { + return value.BsonType switch + { + BsonType.DateTime => DateTime.SpecifyKind(value.ToUniversalTime(), DateTimeKind.Utc), + BsonType.String when DateTimeOffset.TryParse(value.AsString, out var parsed) => parsed.ToUniversalTime(), + _ => null, + }; + } +} + +internal sealed record RedHatCachedFetchMetadata(string? ETag, DateTimeOffset? LastModified); diff --git a/src/StellaOps.Feedser.Source.Distro.RedHat/Internal/RedHatMapper.cs b/src/StellaOps.Feedser.Source.Distro.RedHat/Internal/RedHatMapper.cs new file mode 100644 index 00000000..e9d8a7ad --- /dev/null +++ b/src/StellaOps.Feedser.Source.Distro.RedHat/Internal/RedHatMapper.cs @@ -0,0 +1,758 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text.Json; +using System.Text.Json.Serialization; +using StellaOps.Feedser.Models; +using StellaOps.Feedser.Source.Distro.RedHat.Internal.Models; +using StellaOps.Feedser.Normalization.Cvss; +using StellaOps.Feedser.Normalization.Distro; +using StellaOps.Feedser.Normalization.Identifiers; +using StellaOps.Feedser.Normalization.Text; +using StellaOps.Feedser.Storage.Mongo.Documents; +using StellaOps.Feedser.Storage.Mongo.Dtos; + +namespace StellaOps.Feedser.Source.Distro.RedHat.Internal; + +internal static class RedHatMapper +{ + private static readonly JsonSerializerOptions SerializerOptions = new() + { + PropertyNameCaseInsensitive = true, + NumberHandling = JsonNumberHandling.AllowReadingFromString, + }; + + public static Advisory? Map(string sourceName, DtoRecord dto, DocumentRecord document, JsonDocument payload) + { + ArgumentException.ThrowIfNullOrEmpty(sourceName); + ArgumentNullException.ThrowIfNull(dto); + ArgumentNullException.ThrowIfNull(document); + ArgumentNullException.ThrowIfNull(payload); + + var csaf = JsonSerializer.Deserialize<RedHatCsafEnvelope>(payload.RootElement.GetRawText(), SerializerOptions); + var documentSection = csaf?.Document; + if (documentSection is null) + { + return null; + } + + var tracking = documentSection.Tracking; + var advisoryKey = NormalizeId(tracking?.Id) + ?? NormalizeId(TryGetMetadata(document, "advisoryId")) + ?? NormalizeId(document.Uri) + ?? dto.DocumentId.ToString(); + + var title = !string.IsNullOrWhiteSpace(documentSection.Title) + ? DescriptionNormalizer.Normalize(new[] { new LocalizedText(documentSection.Title, documentSection.Lang) }).Text + : string.Empty; + if (string.IsNullOrEmpty(title)) + { + title = advisoryKey; + } + + var description = NormalizeSummary(documentSection); + var summary = string.IsNullOrEmpty(description.Text) ? null : description.Text; + var severity = NormalizeSeverity(documentSection.AggregateSeverity?.Text); + var published = tracking?.InitialReleaseDate; + var modified = tracking?.CurrentReleaseDate ?? published; + var language = description.Language; + + var aliases = BuildAliases(advisoryKey, csaf); + var references = BuildReferences(sourceName, dto.ValidatedAt, documentSection, csaf); + var productIndex = RedHatProductIndex.Build(csaf.ProductTree); + var affectedPackages = BuildAffectedPackages(sourceName, dto.ValidatedAt, csaf, productIndex); + var cvssMetrics = BuildCvssMetrics(sourceName, dto.ValidatedAt, advisoryKey, csaf); + + var provenance = new[] + { + new AdvisoryProvenance(sourceName, "advisory", advisoryKey, dto.ValidatedAt), + }; + + return new Advisory( + advisoryKey, + title, + summary, + language, + published, + modified, + severity, + exploitKnown: false, + aliases, + references, + affectedPackages, + cvssMetrics, + provenance); + } + + private static IReadOnlyCollection<string> BuildAliases(string advisoryKey, RedHatCsafEnvelope csaf) + { + var aliases = new HashSet<string>(StringComparer.OrdinalIgnoreCase) + { + advisoryKey, + }; + + if (csaf.Vulnerabilities is not null) + { + foreach (var vulnerability in csaf.Vulnerabilities) + { + if (!string.IsNullOrWhiteSpace(vulnerability?.Cve)) + { + aliases.Add(vulnerability!.Cve!.Trim()); + } + } + } + + return aliases; + } + + private static NormalizedDescription NormalizeSummary(RedHatDocumentSection documentSection) + { + var summaryNotes = new List<LocalizedText>(); + var otherNotes = new List<LocalizedText>(); + + if (documentSection.Notes is not null) + { + foreach (var note in documentSection.Notes) + { + if (note is null || string.IsNullOrWhiteSpace(note.Text)) + { + continue; + } + + var candidate = new LocalizedText(note.Text, documentSection.Lang); + if (note.CategoryEquals("summary")) + { + summaryNotes.Add(candidate); + } + else + { + otherNotes.Add(candidate); + } + } + } + + var combined = summaryNotes.Count > 0 + ? summaryNotes.Concat(otherNotes).ToList() + : otherNotes; + + return DescriptionNormalizer.Normalize(combined); + } + + private static IReadOnlyCollection<AdvisoryReference> BuildReferences( + string sourceName, + DateTimeOffset recordedAt, + RedHatDocumentSection? documentSection, + RedHatCsafEnvelope csaf) + { + var references = new List<AdvisoryReference>(); + if (documentSection is not null) + { + AppendReferences(sourceName, recordedAt, documentSection.References, references); + } + + if (csaf.Vulnerabilities is not null) + { + foreach (var vulnerability in csaf.Vulnerabilities) + { + AppendReferences(sourceName, recordedAt, vulnerability?.References, references); + } + } + + return NormalizeReferences(references); + } + + private static void AppendReferences(string sourceName, DateTimeOffset recordedAt, IReadOnlyList<RedHatReference>? items, ICollection<AdvisoryReference> references) + { + if (items is null) + { + return; + } + + foreach (var reference in items) + { + if (reference is null || string.IsNullOrWhiteSpace(reference.Url)) + { + continue; + } + + var url = reference.Url.Trim(); + if (!Validation.LooksLikeHttpUrl(url)) + { + continue; + } + + var provenance = new AdvisoryProvenance(sourceName, "reference", url, recordedAt); + references.Add(new AdvisoryReference(url, reference.Category, null, reference.Summary, provenance)); + } + } + + private static IReadOnlyCollection<AdvisoryReference> NormalizeReferences(IReadOnlyCollection<AdvisoryReference> references) + { + if (references.Count == 0) + { + return Array.Empty<AdvisoryReference>(); + } + + var map = new Dictionary<string, AdvisoryReference>(StringComparer.OrdinalIgnoreCase); + foreach (var reference in references) + { + if (!map.TryGetValue(reference.Url, out var existing)) + { + map[reference.Url] = reference; + continue; + } + + map[reference.Url] = MergeReferences(existing, reference); + } + + return map.Values + .OrderBy(static r => r.Kind is null ? 1 : 0) + .ThenBy(static r => r.Kind ?? string.Empty, StringComparer.OrdinalIgnoreCase) + .ThenBy(static r => r.Url, StringComparer.OrdinalIgnoreCase) + .ThenBy(static r => r.SourceTag ?? string.Empty, StringComparer.OrdinalIgnoreCase) + .ToArray(); + } + + private static AdvisoryReference MergeReferences(AdvisoryReference existing, AdvisoryReference candidate) + { + var kind = existing.Kind ?? candidate.Kind; + var sourceTag = existing.SourceTag ?? candidate.SourceTag; + var summary = ChoosePreferredSummary(existing.Summary, candidate.Summary); + var provenance = existing.Provenance.RecordedAt <= candidate.Provenance.RecordedAt + ? existing.Provenance + : candidate.Provenance; + + if (kind == existing.Kind + && sourceTag == existing.SourceTag + && summary == existing.Summary + && provenance == existing.Provenance) + { + return existing; + } + + if (kind == candidate.Kind + && sourceTag == candidate.SourceTag + && summary == candidate.Summary + && provenance == candidate.Provenance) + { + return candidate; + } + + return new AdvisoryReference(existing.Url, kind, sourceTag, summary, provenance); + } + + private static string? ChoosePreferredSummary(string? left, string? right) + { + var leftValue = string.IsNullOrWhiteSpace(left) ? null : left; + var rightValue = string.IsNullOrWhiteSpace(right) ? null : right; + + if (leftValue is null) + { + return rightValue; + } + + if (rightValue is null) + { + return leftValue; + } + + return leftValue.Length >= rightValue.Length ? leftValue : rightValue; + } + + private static IReadOnlyCollection<AffectedPackage> BuildAffectedPackages( + string sourceName, + DateTimeOffset recordedAt, + RedHatCsafEnvelope csaf, + RedHatProductIndex productIndex) + { + var rpmPackages = new Dictionary<string, RedHatAffectedRpm>(StringComparer.OrdinalIgnoreCase); + var baseProducts = new Dictionary<string, RedHatProductStatusEntry>(StringComparer.OrdinalIgnoreCase); + var knownAffectedByBase = BuildKnownAffectedIndex(csaf); + + if (csaf.Vulnerabilities is not null) + { + foreach (var vulnerability in csaf.Vulnerabilities) + { + if (vulnerability?.ProductStatus is null) + { + continue; + } + + RegisterAll(vulnerability.ProductStatus.Fixed, RedHatProductStatuses.Fixed, productIndex, rpmPackages, baseProducts); + RegisterAll(vulnerability.ProductStatus.FirstFixed, RedHatProductStatuses.FirstFixed, productIndex, rpmPackages, baseProducts); + RegisterAll(vulnerability.ProductStatus.KnownAffected, RedHatProductStatuses.KnownAffected, productIndex, rpmPackages, baseProducts); + RegisterAll(vulnerability.ProductStatus.KnownNotAffected, RedHatProductStatuses.KnownNotAffected, productIndex, rpmPackages, baseProducts); + RegisterAll(vulnerability.ProductStatus.UnderInvestigation, RedHatProductStatuses.UnderInvestigation, productIndex, rpmPackages, baseProducts); + } + } + + var affected = new List<AffectedPackage>(rpmPackages.Count + baseProducts.Count); + + foreach (var rpm in rpmPackages.Values) + { + if (rpm.Statuses.Count == 0) + { + continue; + } + + var ranges = new List<AffectedVersionRange>(); + var statuses = new List<AffectedPackageStatus>(); + var provenance = new AdvisoryProvenance(sourceName, "package.nevra", rpm.ProductId ?? rpm.Nevra, recordedAt); + + var lastKnownAffected = knownAffectedByBase.TryGetValue(rpm.BaseProductId, out var candidate) + ? candidate + : null; + + if (!string.IsNullOrWhiteSpace(lastKnownAffected) + && string.Equals(lastKnownAffected, rpm.Nevra, StringComparison.OrdinalIgnoreCase)) + { + lastKnownAffected = null; + } + + if (rpm.Statuses.Contains(RedHatProductStatuses.Fixed) || rpm.Statuses.Contains(RedHatProductStatuses.FirstFixed)) + { + ranges.Add(new AffectedVersionRange( + "nevra", + introducedVersion: null, + fixedVersion: rpm.Nevra, + lastAffectedVersion: lastKnownAffected, + rangeExpression: null, + provenance: provenance, + primitives: BuildNevraPrimitives(null, rpm.Nevra, lastKnownAffected))); + } + + if (!rpm.Statuses.Contains(RedHatProductStatuses.Fixed) + && !rpm.Statuses.Contains(RedHatProductStatuses.FirstFixed) + && rpm.Statuses.Contains(RedHatProductStatuses.KnownAffected)) + { + ranges.Add(new AffectedVersionRange( + "nevra", + introducedVersion: null, + fixedVersion: null, + lastAffectedVersion: rpm.Nevra, + rangeExpression: null, + provenance: provenance, + primitives: BuildNevraPrimitives(null, null, rpm.Nevra))); + } + + if (rpm.Statuses.Contains(RedHatProductStatuses.KnownNotAffected)) + { + statuses.Add(new AffectedPackageStatus(RedHatProductStatuses.KnownNotAffected, provenance)); + } + + if (rpm.Statuses.Contains(RedHatProductStatuses.UnderInvestigation)) + { + statuses.Add(new AffectedPackageStatus(RedHatProductStatuses.UnderInvestigation, provenance)); + } + + if (ranges.Count == 0 && statuses.Count == 0) + { + continue; + } + + affected.Add(new AffectedPackage( + AffectedPackageTypes.Rpm, + rpm.Nevra, + rpm.Platform, + ranges, + statuses, + new[] { provenance })); + } + + foreach (var baseEntry in baseProducts.Values) + { + if (baseEntry.Statuses.Count == 0) + { + continue; + } + + var node = baseEntry.Node; + if (string.IsNullOrWhiteSpace(node.Cpe)) + { + continue; + } + + if (!IdentifierNormalizer.TryNormalizeCpe(node.Cpe, out var normalizedCpe)) + { + continue; + } + + var provenance = new AdvisoryProvenance(sourceName, "oval", node.ProductId, recordedAt); + var statuses = new List<AffectedPackageStatus>(); + + if (baseEntry.Statuses.Contains(RedHatProductStatuses.KnownAffected)) + { + statuses.Add(new AffectedPackageStatus(RedHatProductStatuses.KnownAffected, provenance)); + } + + if (baseEntry.Statuses.Contains(RedHatProductStatuses.KnownNotAffected)) + { + statuses.Add(new AffectedPackageStatus(RedHatProductStatuses.KnownNotAffected, provenance)); + } + + if (baseEntry.Statuses.Contains(RedHatProductStatuses.UnderInvestigation)) + { + statuses.Add(new AffectedPackageStatus(RedHatProductStatuses.UnderInvestigation, provenance)); + } + + if (statuses.Count == 0) + { + continue; + } + + affected.Add(new AffectedPackage( + AffectedPackageTypes.Cpe, + normalizedCpe!, + node.Name, + Array.Empty<AffectedVersionRange>(), + statuses, + new[] { provenance })); + } + + return affected; + } + + private static Dictionary<string, string> BuildKnownAffectedIndex(RedHatCsafEnvelope csaf) + { + var map = new Dictionary<string, string>(StringComparer.OrdinalIgnoreCase); + if (csaf.Vulnerabilities is null) + { + return map; + } + + foreach (var vulnerability in csaf.Vulnerabilities) + { + var entries = vulnerability?.ProductStatus?.KnownAffected; + if (entries is null) + { + continue; + } + + foreach (var entry in entries) + { + if (string.IsNullOrWhiteSpace(entry)) + { + continue; + } + + var colonIndex = entry.IndexOf(':'); + if (colonIndex <= 0) + { + continue; + } + + var baseId = entry[..colonIndex].Trim(); + if (string.IsNullOrEmpty(baseId)) + { + continue; + } + + var candidate = NormalizeNevra(entry[(colonIndex + 1)..]); + if (!string.IsNullOrEmpty(candidate)) + { + map[baseId] = candidate; + } + } + } + + return map; + } + + private static void RegisterAll( + IReadOnlyList<string>? entries, + string status, + RedHatProductIndex productIndex, + IDictionary<string, RedHatAffectedRpm> rpmPackages, + IDictionary<string, RedHatProductStatusEntry> baseProducts) + { + if (entries is null) + { + return; + } + + foreach (var entry in entries) + { + RegisterProductStatus(entry, status, productIndex, rpmPackages, baseProducts); + } + } + + private static void RegisterProductStatus( + string? rawEntry, + string status, + RedHatProductIndex productIndex, + IDictionary<string, RedHatAffectedRpm> rpmPackages, + IDictionary<string, RedHatProductStatusEntry> baseProducts) + { + if (string.IsNullOrWhiteSpace(rawEntry) || !IsActionableStatus(status)) + { + return; + } + + var entry = rawEntry.Trim(); + var colonIndex = entry.IndexOf(':'); + if (colonIndex <= 0 || colonIndex == entry.Length - 1) + { + if (productIndex.TryGetValue(entry, out var baseOnly)) + { + var aggregate = baseProducts.TryGetValue(baseOnly.ProductId, out var existing) + ? existing + : new RedHatProductStatusEntry(baseOnly); + aggregate.Statuses.Add(status); + baseProducts[baseOnly.ProductId] = aggregate; + } + + return; + } + + var baseId = entry[..colonIndex]; + var packageId = entry[(colonIndex + 1)..]; + + if (productIndex.TryGetValue(baseId, out var baseNode)) + { + var aggregate = baseProducts.TryGetValue(baseNode.ProductId, out var existing) + ? existing + : new RedHatProductStatusEntry(baseNode); + aggregate.Statuses.Add(status); + baseProducts[baseNode.ProductId] = aggregate; + } + + if (!productIndex.TryGetValue(packageId, out var packageNode)) + { + return; + } + + var nevra = NormalizeNevra(packageNode.Name ?? packageNode.ProductId); + if (string.IsNullOrEmpty(nevra)) + { + return; + } + + var platform = baseProducts.TryGetValue(baseId, out var baseEntry) + ? baseEntry.Node.Name ?? baseId + : baseId; + + var key = string.Join('|', nevra, platform ?? string.Empty); + if (!rpmPackages.TryGetValue(key, out var rpm)) + { + rpm = new RedHatAffectedRpm(nevra, baseId, platform, packageNode.ProductId); + rpmPackages[key] = rpm; + } + + rpm.Statuses.Add(status); + } + + private static bool IsActionableStatus(string status) + { + return status.Equals(RedHatProductStatuses.Fixed, StringComparison.OrdinalIgnoreCase) + || status.Equals(RedHatProductStatuses.FirstFixed, StringComparison.OrdinalIgnoreCase) + || status.Equals(RedHatProductStatuses.KnownAffected, StringComparison.OrdinalIgnoreCase) + || status.Equals(RedHatProductStatuses.KnownNotAffected, StringComparison.OrdinalIgnoreCase) + || status.Equals(RedHatProductStatuses.UnderInvestigation, StringComparison.OrdinalIgnoreCase); + } + + private static IReadOnlyCollection<CvssMetric> BuildCvssMetrics( + string sourceName, + DateTimeOffset recordedAt, + string advisoryKey, + RedHatCsafEnvelope csaf) + { + var metrics = new List<CvssMetric>(); + if (csaf.Vulnerabilities is null) + { + return metrics; + } + + foreach (var vulnerability in csaf.Vulnerabilities) + { + if (vulnerability?.Scores is null) + { + continue; + } + + foreach (var score in vulnerability.Scores) + { + var cvss = score?.CvssV3; + if (cvss is null) + { + continue; + } + + if (!CvssMetricNormalizer.TryNormalize(cvss.Version, cvss.VectorString, cvss.BaseScore, cvss.BaseSeverity, out var normalized)) + { + continue; + } + + var provenance = new AdvisoryProvenance(sourceName, "cvss", vulnerability.Cve ?? advisoryKey, recordedAt); + metrics.Add(normalized.ToModel(provenance)); + } + } + + return metrics; + } + + private static string? NormalizeSeverity(string? value) + { + if (string.IsNullOrWhiteSpace(value)) + { + return null; + } + + return value.Trim().ToLowerInvariant() switch + { + "critical" => "critical", + "important" => "high", + "moderate" => "medium", + "low" => "low", + "none" => "none", + _ => value.Trim().ToLowerInvariant(), + }; + } + + private static string? TryGetMetadata(DocumentRecord document, string key) + { + if (document.Metadata is null) + { + return null; + } + + return document.Metadata.TryGetValue(key, out var value) && !string.IsNullOrWhiteSpace(value) + ? value.Trim() + : null; + } + + private static RangePrimitives BuildNevraPrimitives(string? introduced, string? fixedVersion, string? lastAffected) + { + var primitive = new NevraPrimitive( + ParseNevraComponent(introduced), + ParseNevraComponent(fixedVersion), + ParseNevraComponent(lastAffected)); + + return new RangePrimitives(null, primitive, null, null); + } + + private static NevraComponent? ParseNevraComponent(string? value) + { + if (string.IsNullOrWhiteSpace(value)) + { + return null; + } + + if (!Nevra.TryParse(value, out var parsed) || parsed is null) + { + return null; + } + + return new NevraComponent(parsed.Name, parsed.Epoch, parsed.Version, parsed.Release, parsed.Architecture); + } + + private static string? NormalizeId(string? value) + => string.IsNullOrWhiteSpace(value) ? null : value.Trim(); + + private static string NormalizeNevra(string? value) + { + return string.IsNullOrWhiteSpace(value) + ? string.Empty + : value.Trim(); + } +} + +internal sealed class RedHatAffectedRpm +{ + public RedHatAffectedRpm(string nevra, string baseProductId, string? platform, string? productId) + { + Nevra = nevra; + BaseProductId = baseProductId; + Platform = platform; + ProductId = productId; + Statuses = new HashSet<string>(StringComparer.OrdinalIgnoreCase); + } + + public string Nevra { get; } + + public string BaseProductId { get; } + + public string? Platform { get; } + + public string? ProductId { get; } + + public HashSet<string> Statuses { get; } +} + +internal sealed class RedHatProductStatusEntry +{ + public RedHatProductStatusEntry(RedHatProductNode node) + { + Node = node; + Statuses = new HashSet<string>(StringComparer.OrdinalIgnoreCase); + } + + public RedHatProductNode Node { get; } + + public HashSet<string> Statuses { get; } +} + +internal static class RedHatProductStatuses +{ + public const string Fixed = "fixed"; + public const string FirstFixed = "first_fixed"; + public const string KnownAffected = "known_affected"; + public const string KnownNotAffected = "known_not_affected"; + public const string UnderInvestigation = "under_investigation"; +} + +internal sealed class RedHatProductIndex +{ + private readonly Dictionary<string, RedHatProductNode> _products; + + private RedHatProductIndex(Dictionary<string, RedHatProductNode> products) + { + _products = products; + } + + public static RedHatProductIndex Build(RedHatProductTree? tree) + { + var products = new Dictionary<string, RedHatProductNode>(StringComparer.OrdinalIgnoreCase); + if (tree?.Branches is not null) + { + foreach (var branch in tree.Branches) + { + Traverse(branch, products); + } + } + + return new RedHatProductIndex(products); + } + + public bool TryGetValue(string productId, out RedHatProductNode node) + => _products.TryGetValue(productId, out node); + + private static void Traverse(RedHatProductBranch? branch, IDictionary<string, RedHatProductNode> products) + { + if (branch is null) + { + return; + } + + if (branch.Product is not null && !string.IsNullOrWhiteSpace(branch.Product.ProductId)) + { + var id = branch.Product.ProductId.Trim(); + products[id] = new RedHatProductNode( + id, + branch.Product.Name ?? branch.Name ?? id, + branch.Product.ProductIdentificationHelper?.Cpe, + branch.Product.ProductIdentificationHelper?.Purl); + } + + if (branch.Branches is null) + { + return; + } + + foreach (var child in branch.Branches) + { + Traverse(child, products); + } + } +} + +internal sealed record RedHatProductNode(string ProductId, string? Name, string? Cpe, string? Purl); diff --git a/src/StellaOps.Feedser.Source.Distro.RedHat/Internal/RedHatSummaryItem.cs b/src/StellaOps.Feedser.Source.Distro.RedHat/Internal/RedHatSummaryItem.cs new file mode 100644 index 00000000..0e63349d --- /dev/null +++ b/src/StellaOps.Feedser.Source.Distro.RedHat/Internal/RedHatSummaryItem.cs @@ -0,0 +1,66 @@ +using System; +using System.Text.Json; + +namespace StellaOps.Feedser.Source.Distro.RedHat.Internal; + +internal readonly record struct RedHatSummaryItem(string AdvisoryId, DateTimeOffset ReleasedOn, Uri ResourceUri) +{ + private static readonly string[] AdvisoryFields = + { + "RHSA", + "RHBA", + "RHEA", + "RHUI", + "RHBG", + "RHBO", + "advisory" + }; + + public static bool TryParse(JsonElement element, out RedHatSummaryItem item) + { + item = default; + + string? advisoryId = null; + foreach (var field in AdvisoryFields) + { + if (element.TryGetProperty(field, out var advisoryProperty) && advisoryProperty.ValueKind == JsonValueKind.String) + { + var candidate = advisoryProperty.GetString(); + if (!string.IsNullOrWhiteSpace(candidate)) + { + advisoryId = candidate.Trim(); + break; + } + } + } + + if (string.IsNullOrWhiteSpace(advisoryId)) + { + return false; + } + + if (!element.TryGetProperty("released_on", out var releasedProperty) || releasedProperty.ValueKind != JsonValueKind.String) + { + return false; + } + + if (!DateTimeOffset.TryParse(releasedProperty.GetString(), out var releasedOn)) + { + return false; + } + + if (!element.TryGetProperty("resource_url", out var resourceProperty) || resourceProperty.ValueKind != JsonValueKind.String) + { + return false; + } + + var resourceValue = resourceProperty.GetString(); + if (!Uri.TryCreate(resourceValue, UriKind.Absolute, out var resourceUri)) + { + return false; + } + + item = new RedHatSummaryItem(advisoryId!, releasedOn.ToUniversalTime(), resourceUri); + return true; + } +} diff --git a/src/StellaOps.Feedser.Source.Distro.RedHat/Jobs.cs b/src/StellaOps.Feedser.Source.Distro.RedHat/Jobs.cs new file mode 100644 index 00000000..86d93841 --- /dev/null +++ b/src/StellaOps.Feedser.Source.Distro.RedHat/Jobs.cs @@ -0,0 +1,46 @@ +using System; +using System.Threading; +using System.Threading.Tasks; +using StellaOps.Feedser.Core.Jobs; + +namespace StellaOps.Feedser.Source.Distro.RedHat; + +internal static class RedHatJobKinds +{ + public const string Fetch = "source:redhat:fetch"; + public const string Parse = "source:redhat:parse"; + public const string Map = "source:redhat:map"; +} + +internal sealed class RedHatFetchJob : IJob +{ + private readonly RedHatConnector _connector; + + public RedHatFetchJob(RedHatConnector connector) + => _connector = connector ?? throw new ArgumentNullException(nameof(connector)); + + public Task ExecuteAsync(JobExecutionContext context, CancellationToken cancellationToken) + => _connector.FetchAsync(context.Services, cancellationToken); +} + +internal sealed class RedHatParseJob : IJob +{ + private readonly RedHatConnector _connector; + + public RedHatParseJob(RedHatConnector connector) + => _connector = connector ?? throw new ArgumentNullException(nameof(connector)); + + public Task ExecuteAsync(JobExecutionContext context, CancellationToken cancellationToken) + => _connector.ParseAsync(context.Services, cancellationToken); +} + +internal sealed class RedHatMapJob : IJob +{ + private readonly RedHatConnector _connector; + + public RedHatMapJob(RedHatConnector connector) + => _connector = connector ?? throw new ArgumentNullException(nameof(connector)); + + public Task ExecuteAsync(JobExecutionContext context, CancellationToken cancellationToken) + => _connector.MapAsync(context.Services, cancellationToken); +} diff --git a/src/StellaOps.Feedser.Source.Distro.RedHat/Properties/AssemblyInfo.cs b/src/StellaOps.Feedser.Source.Distro.RedHat/Properties/AssemblyInfo.cs new file mode 100644 index 00000000..84fa6ffc --- /dev/null +++ b/src/StellaOps.Feedser.Source.Distro.RedHat/Properties/AssemblyInfo.cs @@ -0,0 +1,3 @@ +using System.Runtime.CompilerServices; + +[assembly: InternalsVisibleTo("StellaOps.Feedser.Source.Distro.RedHat.Tests")] diff --git a/src/StellaOps.Feedser.Source.Distro.RedHat/RedHatConnector.cs b/src/StellaOps.Feedser.Source.Distro.RedHat/RedHatConnector.cs new file mode 100644 index 00000000..f59381bd --- /dev/null +++ b/src/StellaOps.Feedser.Source.Distro.RedHat/RedHatConnector.cs @@ -0,0 +1,434 @@ +using System; +using System.Collections.Generic; +using System.Globalization; +using System.Linq; +using System.Text.Json; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Options; +using MongoDB.Bson; +using MongoDB.Bson.IO; +using StellaOps.Feedser.Models; +using StellaOps.Feedser.Source.Common; +using StellaOps.Feedser.Source.Common.Fetch; +using StellaOps.Feedser.Source.Distro.RedHat.Configuration; +using StellaOps.Feedser.Source.Distro.RedHat.Internal; +using StellaOps.Feedser.Storage.Mongo; +using StellaOps.Feedser.Storage.Mongo.Advisories; +using StellaOps.Feedser.Storage.Mongo.Documents; +using StellaOps.Feedser.Storage.Mongo.Dtos; +using StellaOps.Plugin; + +namespace StellaOps.Feedser.Source.Distro.RedHat; + +public sealed class RedHatConnector : IFeedConnector +{ + private readonly SourceFetchService _fetchService; + private readonly RawDocumentStorage _rawDocumentStorage; + private readonly IDocumentStore _documentStore; + private readonly IDtoStore _dtoStore; + private readonly IAdvisoryStore _advisoryStore; + private readonly ISourceStateRepository _stateRepository; + private readonly ILogger<RedHatConnector> _logger; + private readonly RedHatOptions _options; + private readonly TimeProvider _timeProvider; + + public RedHatConnector( + SourceFetchService fetchService, + RawDocumentStorage rawDocumentStorage, + IDocumentStore documentStore, + IDtoStore dtoStore, + IAdvisoryStore advisoryStore, + ISourceStateRepository stateRepository, + IOptions<RedHatOptions> options, + TimeProvider? timeProvider, + ILogger<RedHatConnector> logger) + { + _fetchService = fetchService ?? throw new ArgumentNullException(nameof(fetchService)); + _rawDocumentStorage = rawDocumentStorage ?? throw new ArgumentNullException(nameof(rawDocumentStorage)); + _documentStore = documentStore ?? throw new ArgumentNullException(nameof(documentStore)); + _dtoStore = dtoStore ?? throw new ArgumentNullException(nameof(dtoStore)); + _advisoryStore = advisoryStore ?? throw new ArgumentNullException(nameof(advisoryStore)); + _stateRepository = stateRepository ?? throw new ArgumentNullException(nameof(stateRepository)); + _options = options?.Value ?? throw new ArgumentNullException(nameof(options)); + _options.Validate(); + _timeProvider = timeProvider ?? TimeProvider.System; + _logger = logger ?? throw new ArgumentNullException(nameof(logger)); + } + + public string SourceName => RedHatConnectorPlugin.SourceName; + + public async Task FetchAsync(IServiceProvider services, CancellationToken cancellationToken) + { + ArgumentNullException.ThrowIfNull(services); + + var cursor = await GetCursorAsync(cancellationToken).ConfigureAwait(false); + var now = _timeProvider.GetUtcNow(); + + var baseline = cursor.LastReleasedOn ?? now - _options.InitialBackfill; + var overlap = _options.Overlap > TimeSpan.Zero ? _options.Overlap : TimeSpan.Zero; + var afterThreshold = baseline - overlap; + if (afterThreshold < DateTimeOffset.UnixEpoch) + { + afterThreshold = DateTimeOffset.UnixEpoch; + } + + ProvenanceDiagnostics.ReportResumeWindow(SourceName, afterThreshold, _logger); + + var processedSet = new HashSet<string>(cursor.ProcessedAdvisoryIds, StringComparer.OrdinalIgnoreCase); + var newSummaries = new List<RedHatSummaryItem>(); + var stopDueToOlderData = false; + var touchedResources = new HashSet<string>(StringComparer.OrdinalIgnoreCase); + + for (var page = 1; page <= _options.MaxPagesPerFetch; page++) + { + var summaryUri = BuildSummaryUri(afterThreshold, page); + var summaryKey = summaryUri.ToString(); + touchedResources.Add(summaryKey); + + var cachedSummary = cursor.TryGetFetchCache(summaryKey); + var summaryMetadata = new Dictionary<string, string>(StringComparer.Ordinal) + { + ["page"] = page.ToString(CultureInfo.InvariantCulture), + ["type"] = "summary" + }; + + var summaryRequest = new SourceFetchRequest(RedHatOptions.HttpClientName, SourceName, summaryUri) + { + Metadata = summaryMetadata, + ETag = cachedSummary?.ETag, + LastModified = cachedSummary?.LastModified, + TimeoutOverride = _options.FetchTimeout, + }; + + SourceFetchContentResult summaryResult; + try + { + summaryResult = await _fetchService.FetchContentAsync(summaryRequest, cancellationToken).ConfigureAwait(false); + } + catch (Exception ex) + { + _logger.LogError(ex, "Red Hat Hydra summary fetch failed for {Uri}", summaryUri); + throw; + } + + if (summaryResult.IsNotModified) + { + if (page == 1) + { + break; + } + + continue; + } + + if (!summaryResult.IsSuccess || summaryResult.Content is null) + { + continue; + } + + cursor = cursor.WithFetchCache(summaryKey, summaryResult.ETag, summaryResult.LastModified); + + using var document = JsonDocument.Parse(summaryResult.Content); + + if (document.RootElement.ValueKind != JsonValueKind.Array) + { + _logger.LogWarning( + "Red Hat Hydra summary response had unexpected payload kind {Kind} for {Uri}", + document.RootElement.ValueKind, + summaryUri); + break; + } + + var pageCount = 0; + foreach (var element in document.RootElement.EnumerateArray()) + { + if (!RedHatSummaryItem.TryParse(element, out var summary)) + { + continue; + } + + pageCount++; + + if (cursor.LastReleasedOn.HasValue) + { + if (summary.ReleasedOn < cursor.LastReleasedOn.Value - overlap) + { + stopDueToOlderData = true; + break; + } + + if (summary.ReleasedOn < cursor.LastReleasedOn.Value) + { + stopDueToOlderData = true; + break; + } + + if (summary.ReleasedOn == cursor.LastReleasedOn.Value && processedSet.Contains(summary.AdvisoryId)) + { + continue; + } + } + + newSummaries.Add(summary); + processedSet.Add(summary.AdvisoryId); + + if (newSummaries.Count >= _options.MaxAdvisoriesPerFetch) + { + break; + } + } + + if (newSummaries.Count >= _options.MaxAdvisoriesPerFetch || stopDueToOlderData) + { + break; + } + + if (pageCount < _options.PageSize) + { + break; + } + } + + if (newSummaries.Count == 0) + { + return; + } + + newSummaries.Sort(static (left, right) => + { + var compare = left.ReleasedOn.CompareTo(right.ReleasedOn); + return compare != 0 + ? compare + : string.CompareOrdinal(left.AdvisoryId, right.AdvisoryId); + }); + + var pendingDocuments = new HashSet<Guid>(cursor.PendingDocuments); + + foreach (var summary in newSummaries) + { + var resourceUri = summary.ResourceUri; + var resourceKey = resourceUri.ToString(); + touchedResources.Add(resourceKey); + + var cached = cursor.TryGetFetchCache(resourceKey); + var metadata = new Dictionary<string, string>(StringComparer.Ordinal) + { + ["advisoryId"] = summary.AdvisoryId, + ["releasedOn"] = summary.ReleasedOn.ToString("O", CultureInfo.InvariantCulture) + }; + + var request = new SourceFetchRequest(RedHatOptions.HttpClientName, SourceName, resourceUri) + { + Metadata = metadata, + ETag = cached?.ETag, + LastModified = cached?.LastModified, + TimeoutOverride = _options.FetchTimeout, + }; + + try + { + var result = await _fetchService.FetchAsync(request, cancellationToken).ConfigureAwait(false); + if (result.IsNotModified) + { + continue; + } + + if (!result.IsSuccess || result.Document is null) + { + continue; + } + + pendingDocuments.Add(result.Document.Id); + cursor = cursor.WithFetchCache(resourceKey, result.Document.Etag, result.Document.LastModified); + } + catch (Exception ex) + { + _logger.LogError(ex, "Red Hat Hydra advisory fetch failed for {Uri}", resourceUri); + throw; + } + } + + var maxRelease = newSummaries.Max(static item => item.ReleasedOn); + var idsForMaxRelease = newSummaries + .Where(item => item.ReleasedOn == maxRelease) + .Select(item => item.AdvisoryId) + .Distinct(StringComparer.OrdinalIgnoreCase) + .ToArray(); + + RedHatCursor updated; + if (cursor.LastReleasedOn.HasValue && maxRelease == cursor.LastReleasedOn.Value) + { + updated = cursor + .WithPendingDocuments(pendingDocuments) + .AddProcessedAdvisories(idsForMaxRelease) + .PruneFetchCache(touchedResources); + } + else + { + updated = cursor + .WithPendingDocuments(pendingDocuments) + .WithLastReleased(maxRelease, idsForMaxRelease) + .PruneFetchCache(touchedResources); + } + + await UpdateCursorAsync(updated, cancellationToken).ConfigureAwait(false); + } + + public async Task ParseAsync(IServiceProvider services, CancellationToken cancellationToken) + { + ArgumentNullException.ThrowIfNull(services); + + var cursor = await GetCursorAsync(cancellationToken).ConfigureAwait(false); + if (cursor.PendingDocuments.Count == 0) + { + return; + } + + var remainingFetch = cursor.PendingDocuments.ToList(); + var pendingMappings = cursor.PendingMappings.ToList(); + + foreach (var documentId in cursor.PendingDocuments) + { + DocumentRecord? document = null; + + try + { + document = await _documentStore.FindAsync(documentId, cancellationToken).ConfigureAwait(false); + if (document is null) + { + remainingFetch.Remove(documentId); + pendingMappings.Remove(documentId); + continue; + } + + if (!document.GridFsId.HasValue) + { + _logger.LogWarning("Red Hat document {DocumentId} missing GridFS content; skipping", document.Id); + remainingFetch.Remove(documentId); + pendingMappings.Remove(documentId); + continue; + } + + var rawBytes = await _rawDocumentStorage.DownloadAsync(document.GridFsId.Value, cancellationToken).ConfigureAwait(false); + using var jsonDocument = JsonDocument.Parse(rawBytes); + var sanitized = JsonSerializer.Serialize(jsonDocument.RootElement); + var payload = BsonDocument.Parse(sanitized); + + var dtoRecord = new DtoRecord( + Guid.NewGuid(), + document.Id, + SourceName, + "redhat.csaf.v2", + payload, + _timeProvider.GetUtcNow()); + + await _dtoStore.UpsertAsync(dtoRecord, cancellationToken).ConfigureAwait(false); + await _documentStore.UpdateStatusAsync(document.Id, DocumentStatuses.PendingMap, cancellationToken).ConfigureAwait(false); + + remainingFetch.Remove(documentId); + if (!pendingMappings.Contains(documentId)) + { + pendingMappings.Add(documentId); + } + } + catch (Exception ex) + { + var uri = document?.Uri ?? documentId.ToString(); + _logger.LogError(ex, "Red Hat CSAF parse failed for {Uri}", uri); + remainingFetch.Remove(documentId); + pendingMappings.Remove(documentId); + } + } + + var updatedCursor = cursor + .WithPendingDocuments(remainingFetch) + .WithPendingMappings(pendingMappings); + + await UpdateCursorAsync(updatedCursor, cancellationToken).ConfigureAwait(false); + } + + public async Task MapAsync(IServiceProvider services, CancellationToken cancellationToken) + { + ArgumentNullException.ThrowIfNull(services); + + var cursor = await GetCursorAsync(cancellationToken).ConfigureAwait(false); + if (cursor.PendingMappings.Count == 0) + { + return; + } + + var pendingMappings = cursor.PendingMappings.ToList(); + + foreach (var documentId in cursor.PendingMappings) + { + try + { + var dto = await _dtoStore.FindByDocumentIdAsync(documentId, cancellationToken).ConfigureAwait(false); + var document = await _documentStore.FindAsync(documentId, cancellationToken).ConfigureAwait(false); + + if (dto is null || document is null) + { + pendingMappings.Remove(documentId); + continue; + } + + var json = dto.Payload.ToJson(new JsonWriterSettings + { + OutputMode = JsonOutputMode.RelaxedExtendedJson, + }); + + using var jsonDocument = JsonDocument.Parse(json); + var advisory = RedHatMapper.Map(SourceName, dto, document, jsonDocument); + if (advisory is null) + { + pendingMappings.Remove(documentId); + continue; + } + + await _advisoryStore.UpsertAsync(advisory, cancellationToken).ConfigureAwait(false); + await _documentStore.UpdateStatusAsync(documentId, DocumentStatuses.Mapped, cancellationToken).ConfigureAwait(false); + pendingMappings.Remove(documentId); + } + catch (Exception ex) + { + _logger.LogError(ex, "Red Hat map failed for document {DocumentId}", documentId); + } + } + + var updatedCursor = cursor.WithPendingMappings(pendingMappings); + await UpdateCursorAsync(updatedCursor, cancellationToken).ConfigureAwait(false); + } + + private async Task<RedHatCursor> GetCursorAsync(CancellationToken cancellationToken) + { + var record = await _stateRepository.TryGetAsync(SourceName, cancellationToken).ConfigureAwait(false); + return RedHatCursor.FromBsonDocument(record?.Cursor); + } + + private async Task UpdateCursorAsync(RedHatCursor cursor, CancellationToken cancellationToken) + { + var completedAt = _timeProvider.GetUtcNow(); + await _stateRepository.UpdateCursorAsync(SourceName, cursor.ToBsonDocument(), completedAt, cancellationToken).ConfigureAwait(false); + } + + private Uri BuildSummaryUri(DateTimeOffset after, int page) + { + var builder = new UriBuilder(_options.BaseEndpoint); + var basePath = builder.Path?.TrimEnd('/') ?? string.Empty; + var summaryPath = _options.SummaryPath.TrimStart('/'); + builder.Path = string.IsNullOrEmpty(basePath) + ? $"/{summaryPath}" + : $"{basePath}/{summaryPath}"; + + var parameters = new Dictionary<string, string>(StringComparer.Ordinal) + { + ["after"] = after.ToString("yyyy-MM-dd", CultureInfo.InvariantCulture), + ["per_page"] = _options.PageSize.ToString(CultureInfo.InvariantCulture), + ["page"] = page.ToString(CultureInfo.InvariantCulture) + }; + + builder.Query = string.Join('&', parameters.Select(static kvp => + $"{Uri.EscapeDataString(kvp.Key)}={Uri.EscapeDataString(kvp.Value)}")); + return builder.Uri; + } +} diff --git a/src/StellaOps.Feedser.Source.Distro.RedHat/RedHatConnectorPlugin.cs b/src/StellaOps.Feedser.Source.Distro.RedHat/RedHatConnectorPlugin.cs new file mode 100644 index 00000000..6b2a03ae --- /dev/null +++ b/src/StellaOps.Feedser.Source.Distro.RedHat/RedHatConnectorPlugin.cs @@ -0,0 +1,19 @@ +using Microsoft.Extensions.DependencyInjection; +using StellaOps.Plugin; + +namespace StellaOps.Feedser.Source.Distro.RedHat; + +public sealed class RedHatConnectorPlugin : IConnectorPlugin +{ + public const string SourceName = "redhat"; + + public string Name => SourceName; + + public bool IsAvailable(IServiceProvider services) => services is not null; + + public IFeedConnector Create(IServiceProvider services) + { + ArgumentNullException.ThrowIfNull(services); + return ActivatorUtilities.CreateInstance<RedHatConnector>(services); + } +} diff --git a/src/StellaOps.Feedser.Source.Distro.RedHat/RedHatDependencyInjectionRoutine.cs b/src/StellaOps.Feedser.Source.Distro.RedHat/RedHatDependencyInjectionRoutine.cs new file mode 100644 index 00000000..39574db0 --- /dev/null +++ b/src/StellaOps.Feedser.Source.Distro.RedHat/RedHatDependencyInjectionRoutine.cs @@ -0,0 +1,54 @@ +using System; +using Microsoft.Extensions.Configuration; +using Microsoft.Extensions.DependencyInjection; +using StellaOps.DependencyInjection; +using StellaOps.Feedser.Core.Jobs; +using StellaOps.Feedser.Source.Distro.RedHat.Configuration; + +namespace StellaOps.Feedser.Source.Distro.RedHat; + +public sealed class RedHatDependencyInjectionRoutine : IDependencyInjectionRoutine +{ + private const string ConfigurationSection = "feedser:sources:redhat"; + private const string FetchCron = "0,15,30,45 * * * *"; + private const string ParseCron = "5,20,35,50 * * * *"; + private const string MapCron = "10,25,40,55 * * * *"; + + private static readonly TimeSpan FetchTimeout = TimeSpan.FromMinutes(12); + private static readonly TimeSpan ParseTimeout = TimeSpan.FromMinutes(15); + private static readonly TimeSpan MapTimeout = TimeSpan.FromMinutes(20); + private static readonly TimeSpan LeaseDuration = TimeSpan.FromMinutes(6); + + public IServiceCollection Register(IServiceCollection services, IConfiguration configuration) + { + ArgumentNullException.ThrowIfNull(services); + ArgumentNullException.ThrowIfNull(configuration); + + services.AddRedHatConnector(options => + { + configuration.GetSection(ConfigurationSection).Bind(options); + options.Validate(); + }); + + var schedulerBuilder = new JobSchedulerBuilder(services); + + schedulerBuilder + .AddJob<RedHatFetchJob>( + RedHatJobKinds.Fetch, + cronExpression: FetchCron, + timeout: FetchTimeout, + leaseDuration: LeaseDuration) + .AddJob<RedHatParseJob>( + RedHatJobKinds.Parse, + cronExpression: ParseCron, + timeout: ParseTimeout, + leaseDuration: LeaseDuration) + .AddJob<RedHatMapJob>( + RedHatJobKinds.Map, + cronExpression: MapCron, + timeout: MapTimeout, + leaseDuration: LeaseDuration); + + return services; + } +} diff --git a/src/StellaOps.Feedser.Source.Distro.RedHat/RedHatServiceCollectionExtensions.cs b/src/StellaOps.Feedser.Source.Distro.RedHat/RedHatServiceCollectionExtensions.cs new file mode 100644 index 00000000..5cf81f46 --- /dev/null +++ b/src/StellaOps.Feedser.Source.Distro.RedHat/RedHatServiceCollectionExtensions.cs @@ -0,0 +1,34 @@ +using System; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Options; +using StellaOps.Feedser.Source.Common.Http; +using StellaOps.Feedser.Source.Distro.RedHat.Configuration; + +namespace StellaOps.Feedser.Source.Distro.RedHat; + +public static class RedHatServiceCollectionExtensions +{ + public static IServiceCollection AddRedHatConnector(this IServiceCollection services, Action<RedHatOptions> configure) + { + ArgumentNullException.ThrowIfNull(services); + ArgumentNullException.ThrowIfNull(configure); + + services.AddOptions<RedHatOptions>() + .Configure(configure) + .PostConfigure(static opts => opts.Validate()); + + services.AddSourceHttpClient(RedHatOptions.HttpClientName, (sp, httpOptions) => + { + var options = sp.GetRequiredService<IOptions<RedHatOptions>>().Value; + httpOptions.BaseAddress = options.BaseEndpoint; + httpOptions.Timeout = options.FetchTimeout; + httpOptions.UserAgent = options.UserAgent; + httpOptions.AllowedHosts.Clear(); + httpOptions.AllowedHosts.Add(options.BaseEndpoint.Host); + httpOptions.DefaultRequestHeaders["Accept"] = "application/json"; + }); + + services.AddTransient<RedHatConnector>(); + return services; + } +} diff --git a/src/StellaOps.Feedser.Source.Distro.RedHat/StellaOps.Feedser.Source.Distro.RedHat.csproj b/src/StellaOps.Feedser.Source.Distro.RedHat/StellaOps.Feedser.Source.Distro.RedHat.csproj new file mode 100644 index 00000000..bedbc3b9 --- /dev/null +++ b/src/StellaOps.Feedser.Source.Distro.RedHat/StellaOps.Feedser.Source.Distro.RedHat.csproj @@ -0,0 +1,15 @@ +<Project Sdk="Microsoft.NET.Sdk"> + <PropertyGroup> + <TargetFramework>net10.0</TargetFramework> + <ImplicitUsings>enable</ImplicitUsings> + <Nullable>enable</Nullable> + </PropertyGroup> + <ItemGroup> + <ProjectReference Include="../StellaOps.Plugin/StellaOps.Plugin.csproj" /> + <ProjectReference Include="../StellaOps.DependencyInjection/StellaOps.DependencyInjection.csproj" /> + <ProjectReference Include="..\StellaOps.Feedser.Source.Common\StellaOps.Feedser.Source.Common.csproj" /> + <ProjectReference Include="..\StellaOps.Feedser.Models\StellaOps.Feedser.Models.csproj" /> + <ProjectReference Include="..\StellaOps.Feedser.Storage.Mongo\StellaOps.Feedser.Storage.Mongo.csproj" /> + <ProjectReference Include="..\StellaOps.Feedser.Normalization\StellaOps.Feedser.Normalization.csproj" /> + </ItemGroup> +</Project> diff --git a/src/StellaOps.Feedser.Source.Distro.RedHat/TASKS.md b/src/StellaOps.Feedser.Source.Distro.RedHat/TASKS.md new file mode 100644 index 00000000..2c431a34 --- /dev/null +++ b/src/StellaOps.Feedser.Source.Distro.RedHat/TASKS.md @@ -0,0 +1,16 @@ +# TASKS +| Task | Owner(s) | Depends on | Notes | +|---|---|---|---| +|Hydra fetch with after= cursor|BE-Conn-RH|Source.Common|**DONE** – windowed paging with overlap, ETag/Last-Modified persisted.| +|DTOs for Security Data + OVAL|BE-Conn-RH|Tests|**DONE** – CSAF payloads serialized into `redhat.csaf.v2` DTOs.| +|NEVRA parser/comparer (complete)|BE-Conn-RH|Models|**DONE** – parser/comparer shipped with coverage; add edge cases as needed.| +|Mapper to canonical rpm/cpe affected|BE-Conn-RH|Models|**DONE** – maps fixed/known ranges, CPE provenance, status ranges.| +|Job scheduler registration aligns with Options pipeline|BE-Conn-RH|Core|**DONE** – registered fetch/parse/map via JobSchedulerBuilder, preserving option overrides and tightening cron/timeouts.| +|Watermark persistence + resume|BE-Conn-RH|Storage.Mongo|**DONE** – cursor updates via SourceStateRepository.| +|Precedence tests vs NVD|QA|Merge|**DONE** – Added AffectedPackagePrecedenceResolver + tests ensuring Red Hat CPEs override NVD ranges.| +|Golden mapping fixtures|QA|Fixtures|**DONE** – fixture validation test now snapshots RHSA-2025:0001/0002/0003 with env-driven regeneration.| +|Job scheduling defaults for source:redhat tasks|BE-Core|JobScheduler|**DONE** – Cron windows + per-job timeouts defined for fetch/parse/map.| +|Express unaffected/investigation statuses without overloading range fields|BE-Conn-RH|Models|**DONE** – Introduced AffectedPackageStatus collection and updated mapper/tests.| +|Reference dedupe & ordering in mapper|BE-Conn-RH|Models|DONE – mapper consolidates by URL, merges metadata, deterministic ordering validated in tests.| +|Hydra summary fetch through SourceFetchService|BE-Conn-RH|Source.Common|DONE – summary pages now fetched via SourceFetchService with cache + conditional headers.| +|Fixture validation sweep|QA|Testing|**DOING (2025-10-10)** – Regenerate RHSA fixtures once mapper fixes land, review snapshot diffs, and update docs; blocked by outstanding range provenance patches.| diff --git a/src/StellaOps.Feedser.Source.Distro.Suse.Tests/Source/Distro/Suse/Fixtures/suse-changes.csv b/src/StellaOps.Feedser.Source.Distro.Suse.Tests/Source/Distro/Suse/Fixtures/suse-changes.csv new file mode 100644 index 00000000..646fc46a --- /dev/null +++ b/src/StellaOps.Feedser.Source.Distro.Suse.Tests/Source/Distro/Suse/Fixtures/suse-changes.csv @@ -0,0 +1,2 @@ +"suse-su-2025_0001-1.json","2025-01-21T10:00:00Z" +"suse-su-2025_0002-1.json","2025-01-22T08:30:00Z" diff --git a/src/StellaOps.Feedser.Source.Distro.Suse.Tests/Source/Distro/Suse/Fixtures/suse-su-2025_0001-1.json b/src/StellaOps.Feedser.Source.Distro.Suse.Tests/Source/Distro/Suse/Fixtures/suse-su-2025_0001-1.json new file mode 100644 index 00000000..50d05bd2 --- /dev/null +++ b/src/StellaOps.Feedser.Source.Distro.Suse.Tests/Source/Distro/Suse/Fixtures/suse-su-2025_0001-1.json @@ -0,0 +1,63 @@ +{ + "document": { + "title": "openssl - security update", + "tracking": { + "id": "SUSE-SU-2025:0001-1", + "initial_release_date": "2025-01-21T00:00:00Z", + "current_release_date": "2025-01-21T00:00:00Z" + }, + "references": [ + { + "category": "self", + "summary": "SUSE notice", + "url": "https://www.suse.com/security/cve/CVE-2025-0001/" + } + ], + "notes": [ + { + "category": "summary", + "text": "Security update for openssl" + } + ] + }, + "product_tree": { + "branches": [ + { + "category": "vendor", + "name": "SUSE", + "branches": [ + { + "category": "product_family", + "name": "SUSE Linux Enterprise Server 15 SP5", + "branches": [ + { + "category": "architecture", + "name": "x86_64", + "branches": [ + { + "category": "product_version", + "name": "openssl-1.1.1w-150500.17.25.1.x86_64", + "product": { + "name": "openssl-1.1.1w-150500.17.25.1.x86_64", + "product_id": "SUSE Linux Enterprise Server 15 SP5:openssl-1.1.1w-150500.17.25.1.x86_64" + } + } + ] + } + ] + } + ] + } + ] + }, + "vulnerabilities": [ + { + "cve": "CVE-2025-0001", + "product_status": { + "recommended": [ + "SUSE Linux Enterprise Server 15 SP5:openssl-1.1.1w-150500.17.25.1.x86_64" + ] + } + } + ] +} diff --git a/src/StellaOps.Feedser.Source.Distro.Suse.Tests/Source/Distro/Suse/Fixtures/suse-su-2025_0002-1.json b/src/StellaOps.Feedser.Source.Distro.Suse.Tests/Source/Distro/Suse/Fixtures/suse-su-2025_0002-1.json new file mode 100644 index 00000000..fb40fa48 --- /dev/null +++ b/src/StellaOps.Feedser.Source.Distro.Suse.Tests/Source/Distro/Suse/Fixtures/suse-su-2025_0002-1.json @@ -0,0 +1,66 @@ +{ + "document": { + "title": "postgresql - investigation update", + "tracking": { + "id": "SUSE-SU-2025:0002-1", + "initial_release_date": "2025-01-22T00:00:00Z", + "current_release_date": "2025-01-22T00:00:00Z" + }, + "references": [ + { + "category": "external", + "summary": "Upstream CVE", + "url": "https://www.postgresql.org/support/security/CVE-2025-0002/" + } + ], + "notes": [ + { + "category": "summary", + "text": "Investigation ongoing for postgresql security issue." + } + ] + }, + "product_tree": { + "branches": [ + { + "category": "vendor", + "name": "SUSE", + "branches": [ + { + "category": "product_family", + "name": "openSUSE Tumbleweed", + "branches": [ + { + "category": "architecture", + "name": "x86_64", + "branches": [ + { + "category": "product_version", + "name": "postgresql16-16.3-2.1.x86_64", + "product": { + "name": "postgresql16-16.3-2.1.x86_64", + "product_id": "openSUSE Tumbleweed:postgresql16-16.3-2.1.x86_64" + } + } + ] + } + ] + } + ] + } + ] + }, + "vulnerabilities": [ + { + "cve": "CVE-2025-0002", + "product_status": { + "known_affected": [ + "openSUSE Tumbleweed:postgresql16-16.3-2.1.x86_64" + ], + "under_investigation": [ + "openSUSE Tumbleweed:postgresql16-16.3-2.1.x86_64" + ] + } + } + ] +} diff --git a/src/StellaOps.Feedser.Source.Distro.Suse.Tests/StellaOps.Feedser.Source.Distro.Suse.Tests.csproj b/src/StellaOps.Feedser.Source.Distro.Suse.Tests/StellaOps.Feedser.Source.Distro.Suse.Tests.csproj new file mode 100644 index 00000000..f56d2dff --- /dev/null +++ b/src/StellaOps.Feedser.Source.Distro.Suse.Tests/StellaOps.Feedser.Source.Distro.Suse.Tests.csproj @@ -0,0 +1,18 @@ +<Project Sdk="Microsoft.NET.Sdk"> + <PropertyGroup> + <TargetFramework>net10.0</TargetFramework> + <ImplicitUsings>enable</ImplicitUsings> + <Nullable>enable</Nullable> + </PropertyGroup> + <ItemGroup> + <ProjectReference Include="../StellaOps.Feedser.Source.Distro.Suse/StellaOps.Feedser.Source.Distro.Suse.csproj" /> + <ProjectReference Include="../StellaOps.Feedser.Source.Common/StellaOps.Feedser.Source.Common.csproj" /> + <ProjectReference Include="../StellaOps.Feedser.Models/StellaOps.Feedser.Models.csproj" /> + <ProjectReference Include="../StellaOps.Feedser.Storage.Mongo/StellaOps.Feedser.Storage.Mongo.csproj" /> + </ItemGroup> + <ItemGroup> + <None Update="Source\Distro\Suse\Fixtures\**\*"> + <CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory> + </None> + </ItemGroup> +</Project> diff --git a/src/StellaOps.Feedser.Source.Distro.Suse.Tests/SuseConnectorTests.cs b/src/StellaOps.Feedser.Source.Distro.Suse.Tests/SuseConnectorTests.cs new file mode 100644 index 00000000..765fa856 --- /dev/null +++ b/src/StellaOps.Feedser.Source.Distro.Suse.Tests/SuseConnectorTests.cs @@ -0,0 +1,168 @@ +using System; +using System.IO; +using System.Linq; +using System.Net; +using System.Net.Http; +using System.Net.Http.Headers; +using System.Text; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Http; +using Microsoft.Extensions.Logging.Abstractions; +using Microsoft.Extensions.Time.Testing; +using StellaOps.Feedser.Models; +using StellaOps.Feedser.Source.Common; +using StellaOps.Feedser.Source.Common.Testing; +using StellaOps.Feedser.Source.Distro.Suse; +using StellaOps.Feedser.Source.Distro.Suse.Configuration; +using StellaOps.Feedser.Storage.Mongo; +using StellaOps.Feedser.Storage.Mongo.Advisories; +using StellaOps.Feedser.Storage.Mongo.Documents; +using StellaOps.Feedser.Storage.Mongo.Dtos; +using StellaOps.Feedser.Testing; +using Xunit; +using Xunit.Abstractions; + +namespace StellaOps.Feedser.Source.Distro.Suse.Tests; + +[Collection("mongo-fixture")] +public sealed class SuseConnectorTests : IAsyncLifetime +{ + private static readonly Uri ChangesUri = new("https://ftp.suse.com/pub/projects/security/csaf/changes.csv"); + private static readonly Uri AdvisoryResolvedUri = new("https://ftp.suse.com/pub/projects/security/csaf/suse-su-2025_0001-1.json"); + private static readonly Uri AdvisoryOpenUri = new("https://ftp.suse.com/pub/projects/security/csaf/suse-su-2025_0002-1.json"); + + private readonly MongoIntegrationFixture _fixture; + private readonly FakeTimeProvider _timeProvider; + private readonly CannedHttpMessageHandler _handler; + + public SuseConnectorTests(MongoIntegrationFixture fixture, ITestOutputHelper output) + { + _fixture = fixture; + _timeProvider = new FakeTimeProvider(new DateTimeOffset(2025, 1, 22, 0, 0, 0, TimeSpan.Zero)); + _handler = new CannedHttpMessageHandler(); + } + + [Fact] + public async Task FetchParseMap_ProcessesResolvedAndOpenNotices() + { + await using var provider = await BuildServiceProviderAsync(); + + SeedInitialResponses(); + + var connector = provider.GetRequiredService<SuseConnector>(); + await connector.FetchAsync(provider, CancellationToken.None); + _timeProvider.Advance(TimeSpan.FromMinutes(1)); + await connector.ParseAsync(provider, CancellationToken.None); + await connector.MapAsync(provider, CancellationToken.None); + + var advisoryStore = provider.GetRequiredService<IAdvisoryStore>(); + var advisories = await advisoryStore.GetRecentAsync(10, CancellationToken.None); + Assert.Equal(2, advisories.Count); + + var resolved = advisories.Single(a => a.AdvisoryKey == "SUSE-SU-2025:0001-1"); + var resolvedPackage = Assert.Single(resolved.AffectedPackages); + var resolvedRange = Assert.Single(resolvedPackage.VersionRanges); + Assert.Equal("nevra", resolvedRange.RangeKind); + Assert.NotNull(resolvedRange.Primitives); + Assert.NotNull(resolvedRange.Primitives!.Nevra?.Fixed); + + var open = advisories.Single(a => a.AdvisoryKey == "SUSE-SU-2025:0002-1"); + var openPackage = Assert.Single(open.AffectedPackages); + Assert.Equal("open", openPackage.Statuses.Single().Status); + + SeedNotModifiedResponses(); + + await connector.FetchAsync(provider, CancellationToken.None); + _timeProvider.Advance(TimeSpan.FromMinutes(1)); + await connector.ParseAsync(provider, CancellationToken.None); + await connector.MapAsync(provider, CancellationToken.None); + + advisories = await advisoryStore.GetRecentAsync(10, CancellationToken.None); + Assert.Equal(2, advisories.Count); + _handler.AssertNoPendingResponses(); + } + + private async Task<ServiceProvider> BuildServiceProviderAsync() + { + await _fixture.Client.DropDatabaseAsync(_fixture.Database.DatabaseNamespace.DatabaseName); + _handler.Clear(); + + var services = new ServiceCollection(); + services.AddLogging(builder => builder.AddProvider(NullLoggerProvider.Instance)); + services.AddSingleton<TimeProvider>(_timeProvider); + services.AddSingleton(_handler); + + services.AddMongoStorage(options => + { + options.ConnectionString = _fixture.Runner.ConnectionString; + options.DatabaseName = _fixture.Database.DatabaseNamespace.DatabaseName; + options.CommandTimeout = TimeSpan.FromSeconds(5); + }); + + services.AddSourceCommon(); + services.AddSuseConnector(options => + { + options.ChangesEndpoint = ChangesUri; + options.AdvisoryBaseUri = new Uri("https://ftp.suse.com/pub/projects/security/csaf/"); + options.MaxAdvisoriesPerFetch = 5; + options.RequestDelay = TimeSpan.Zero; + }); + + services.Configure<HttpClientFactoryOptions>(SuseOptions.HttpClientName, builderOptions => + { + builderOptions.HttpMessageHandlerBuilderActions.Add(builder => + { + builder.PrimaryHandler = _handler; + }); + }); + + var provider = services.BuildServiceProvider(); + var bootstrapper = provider.GetRequiredService<MongoBootstrapper>(); + await bootstrapper.InitializeAsync(CancellationToken.None); + return provider; + } + + private void SeedInitialResponses() + { + _handler.AddResponse(ChangesUri, () => BuildResponse(HttpStatusCode.OK, "suse-changes.csv", "\"changes-v1\"")); + _handler.AddResponse(AdvisoryResolvedUri, () => BuildResponse(HttpStatusCode.OK, "suse-su-2025_0001-1.json", "\"adv-1\"")); + _handler.AddResponse(AdvisoryOpenUri, () => BuildResponse(HttpStatusCode.OK, "suse-su-2025_0002-1.json", "\"adv-2\"")); + } + + private void SeedNotModifiedResponses() + { + _handler.AddResponse(ChangesUri, () => BuildResponse(HttpStatusCode.NotModified, "suse-changes.csv", "\"changes-v1\"")); + _handler.AddResponse(AdvisoryResolvedUri, () => BuildResponse(HttpStatusCode.NotModified, "suse-su-2025_0001-1.json", "\"adv-1\"")); + _handler.AddResponse(AdvisoryOpenUri, () => BuildResponse(HttpStatusCode.NotModified, "suse-su-2025_0002-1.json", "\"adv-2\"")); + } + + private HttpResponseMessage BuildResponse(HttpStatusCode statusCode, string fixture, string etag) + { + var response = new HttpResponseMessage(statusCode); + if (statusCode == HttpStatusCode.OK) + { + var contentType = fixture.EndsWith(".csv", StringComparison.OrdinalIgnoreCase) ? "text/csv" : "application/json"; + response.Content = new StringContent(ReadFixture(Path.Combine("Source", "Distro", "Suse", "Fixtures", fixture)), Encoding.UTF8, contentType); + } + + response.Headers.ETag = new EntityTagHeaderValue(etag); + return response; + } + + private static string ReadFixture(string relativePath) + { + var path = Path.Combine(AppContext.BaseDirectory, relativePath.Replace('/', Path.DirectorySeparatorChar)); + if (!File.Exists(path)) + { + throw new FileNotFoundException($"Fixture '{relativePath}' not found.", path); + } + + return File.ReadAllText(path); + } + + public Task InitializeAsync() => Task.CompletedTask; + + public Task DisposeAsync() => Task.CompletedTask; +} diff --git a/src/StellaOps.Feedser.Source.Distro.Suse.Tests/SuseCsafParserTests.cs b/src/StellaOps.Feedser.Source.Distro.Suse.Tests/SuseCsafParserTests.cs new file mode 100644 index 00000000..fdde44fc --- /dev/null +++ b/src/StellaOps.Feedser.Source.Distro.Suse.Tests/SuseCsafParserTests.cs @@ -0,0 +1,52 @@ +using System; +using System.IO; +using System.Linq; +using System.Text.Json; +using StellaOps.Feedser.Source.Distro.Suse.Internal; +using Xunit; + +namespace StellaOps.Feedser.Source.Distro.Suse.Tests; + +public sealed class SuseCsafParserTests +{ + [Fact] + public void Parse_ProducesRecommendedAndAffectedPackages() + { + var json = ReadFixture("Source/Distro/Suse/Fixtures/suse-su-2025_0001-1.json"); + var dto = SuseCsafParser.Parse(json); + + Assert.Equal("SUSE-SU-2025:0001-1", dto.AdvisoryId); + Assert.Contains("CVE-2025-0001", dto.CveIds); + var package = Assert.Single(dto.Packages); + Assert.Equal("openssl", package.Package); + Assert.Equal("resolved", package.Status); + Assert.NotNull(package.FixedVersion); + Assert.Equal("SUSE Linux Enterprise Server 15 SP5", package.Platform); + Assert.Equal("openssl-1.1.1w-150500.17.25.1.x86_64", package.CanonicalNevra); + } + + [Fact] + public void Parse_HandlesOpenInvestigation() + { + var json = ReadFixture("Source/Distro/Suse/Fixtures/suse-su-2025_0002-1.json"); + var dto = SuseCsafParser.Parse(json); + + Assert.Equal("SUSE-SU-2025:0002-1", dto.AdvisoryId); + Assert.Contains("CVE-2025-0002", dto.CveIds); + var package = Assert.Single(dto.Packages); + Assert.Equal("open", package.Status); + Assert.Equal("postgresql16", package.Package); + Assert.NotNull(package.LastAffectedVersion); + } + + private static string ReadFixture(string relativePath) + { + var path = Path.Combine(AppContext.BaseDirectory, relativePath.Replace('/', Path.DirectorySeparatorChar)); + if (!File.Exists(path)) + { + throw new FileNotFoundException($"Fixture '{relativePath}' not found.", path); + } + + return File.ReadAllText(path); + } +} diff --git a/src/StellaOps.Feedser.Source.Distro.Suse.Tests/SuseMapperTests.cs b/src/StellaOps.Feedser.Source.Distro.Suse.Tests/SuseMapperTests.cs new file mode 100644 index 00000000..09fbc180 --- /dev/null +++ b/src/StellaOps.Feedser.Source.Distro.Suse.Tests/SuseMapperTests.cs @@ -0,0 +1,52 @@ +using System; +using System.Collections.Generic; +using System.IO; +using MongoDB.Bson; +using StellaOps.Feedser.Models; +using StellaOps.Feedser.Source.Common; +using StellaOps.Feedser.Source.Distro.Suse; +using StellaOps.Feedser.Source.Distro.Suse.Internal; +using StellaOps.Feedser.Storage.Mongo.Documents; +using Xunit; + +namespace StellaOps.Feedser.Source.Distro.Suse.Tests; + +public sealed class SuseMapperTests +{ + [Fact] + public void Map_BuildsNevraRangePrimitives() + { + var json = File.ReadAllText(Path.Combine(AppContext.BaseDirectory, "Source", "Distro", "Suse", "Fixtures", "suse-su-2025_0001-1.json")); + var dto = SuseCsafParser.Parse(json); + + var document = new DocumentRecord( + Guid.NewGuid(), + SuseConnectorPlugin.SourceName, + "https://ftp.suse.com/pub/projects/security/csaf/suse-su-2025_0001-1.json", + DateTimeOffset.UtcNow, + "sha256", + DocumentStatuses.PendingParse, + "application/json", + Headers: null, + Metadata: new Dictionary<string, string>(StringComparer.Ordinal) + { + ["suse.id"] = dto.AdvisoryId + }, + Etag: "adv-1", + LastModified: DateTimeOffset.UtcNow, + GridFsId: ObjectId.Empty); + + var mapped = SuseMapper.Map(dto, document, DateTimeOffset.UtcNow); + + Assert.Equal(dto.AdvisoryId, mapped.AdvisoryKey); + var package = Assert.Single(mapped.AffectedPackages); + Assert.Equal(AffectedPackageTypes.Rpm, package.Type); + var range = Assert.Single(package.VersionRanges); + Assert.Equal("nevra", range.RangeKind); + Assert.NotNull(range.Primitives); + Assert.NotNull(range.Primitives!.Nevra); + Assert.NotNull(range.Primitives.Nevra!.Fixed); + Assert.Equal("openssl", range.Primitives.Nevra.Fixed!.Name); + Assert.Equal("SUSE Linux Enterprise Server 15 SP5", package.Platform); + } +} diff --git a/src/StellaOps.Feedser.Source.Distro.Suse/AssemblyInfo.cs b/src/StellaOps.Feedser.Source.Distro.Suse/AssemblyInfo.cs new file mode 100644 index 00000000..0a90994d --- /dev/null +++ b/src/StellaOps.Feedser.Source.Distro.Suse/AssemblyInfo.cs @@ -0,0 +1,3 @@ +using System.Runtime.CompilerServices; + +[assembly: InternalsVisibleTo("StellaOps.Feedser.Source.Distro.Suse.Tests")] diff --git a/src/StellaOps.Feedser.Source.Distro.Suse/Configuration/SuseOptions.cs b/src/StellaOps.Feedser.Source.Distro.Suse/Configuration/SuseOptions.cs new file mode 100644 index 00000000..b1849d96 --- /dev/null +++ b/src/StellaOps.Feedser.Source.Distro.Suse/Configuration/SuseOptions.cs @@ -0,0 +1,86 @@ +using System; + +namespace StellaOps.Feedser.Source.Distro.Suse.Configuration; + +public sealed class SuseOptions +{ + public const string HttpClientName = "feedser.suse"; + + /// <summary> + /// CSV index enumerating CSAF advisories with their last modification timestamps. + /// </summary> + public Uri ChangesEndpoint { get; set; } = new("https://ftp.suse.com/pub/projects/security/csaf/changes.csv"); + + /// <summary> + /// Base URI where individual CSAF advisories reside (filename appended verbatim). + /// </summary> + public Uri AdvisoryBaseUri { get; set; } = new("https://ftp.suse.com/pub/projects/security/csaf/"); + + /// <summary> + /// Maximum advisories to fetch per run to bound backfill effort. + /// </summary> + public int MaxAdvisoriesPerFetch { get; set; } = 40; + + /// <summary> + /// Initial history window for first-time execution. + /// </summary> + public TimeSpan InitialBackfill { get; set; } = TimeSpan.FromDays(30); + + /// <summary> + /// Overlap window applied when resuming to capture late edits. + /// </summary> + public TimeSpan ResumeOverlap { get; set; } = TimeSpan.FromDays(3); + + /// <summary> + /// Optional delay between advisory detail fetches. + /// </summary> + public TimeSpan RequestDelay { get; set; } = TimeSpan.Zero; + + /// <summary> + /// Custom user agent presented to SUSE endpoints. + /// </summary> + public string UserAgent { get; set; } = "StellaOps.Feedser.Suse/0.1 (+https://stella-ops.org)"; + + /// <summary> + /// Timeout override applied to HTTP requests (defaults to 60 seconds when unset). + /// </summary> + public TimeSpan FetchTimeout { get; set; } = TimeSpan.FromSeconds(45); + + public void Validate() + { + if (ChangesEndpoint is null || !ChangesEndpoint.IsAbsoluteUri) + { + throw new InvalidOperationException("SuseOptions.ChangesEndpoint must be an absolute URI."); + } + + if (AdvisoryBaseUri is null || !AdvisoryBaseUri.IsAbsoluteUri) + { + throw new InvalidOperationException("SuseOptions.AdvisoryBaseUri must be an absolute URI."); + } + + if (MaxAdvisoriesPerFetch <= 0 || MaxAdvisoriesPerFetch > 250) + { + throw new InvalidOperationException("MaxAdvisoriesPerFetch must be between 1 and 250."); + } + + if (InitialBackfill < TimeSpan.Zero || InitialBackfill > TimeSpan.FromDays(365)) + { + throw new InvalidOperationException("InitialBackfill must be between 0 and 365 days."); + } + + if (ResumeOverlap < TimeSpan.Zero || ResumeOverlap > TimeSpan.FromDays(14)) + { + throw new InvalidOperationException("ResumeOverlap must be between 0 and 14 days."); + } + + if (FetchTimeout <= TimeSpan.Zero || FetchTimeout > TimeSpan.FromMinutes(5)) + { + throw new InvalidOperationException("FetchTimeout must be positive and less than five minutes."); + } + + if (RequestDelay < TimeSpan.Zero || RequestDelay > TimeSpan.FromSeconds(10)) + { + throw new InvalidOperationException("RequestDelay must be between 0 and 10 seconds."); + } + } +} diff --git a/src/StellaOps.Feedser.Source.Distro.Suse/Internal/SuseAdvisoryDto.cs b/src/StellaOps.Feedser.Source.Distro.Suse/Internal/SuseAdvisoryDto.cs new file mode 100644 index 00000000..a79b35b8 --- /dev/null +++ b/src/StellaOps.Feedser.Source.Distro.Suse/Internal/SuseAdvisoryDto.cs @@ -0,0 +1,28 @@ +using System; +using System.Collections.Generic; + +namespace StellaOps.Feedser.Source.Distro.Suse.Internal; + +internal sealed record SuseAdvisoryDto( + string AdvisoryId, + string Title, + string? Summary, + DateTimeOffset Published, + IReadOnlyList<string> CveIds, + IReadOnlyList<SusePackageStateDto> Packages, + IReadOnlyList<SuseReferenceDto> References); + +internal sealed record SusePackageStateDto( + string Package, + string Platform, + string? Architecture, + string CanonicalNevra, + string? IntroducedVersion, + string? FixedVersion, + string? LastAffectedVersion, + string Status); + +internal sealed record SuseReferenceDto( + string Url, + string? Kind, + string? Title); diff --git a/src/StellaOps.Feedser.Source.Distro.Suse/Internal/SuseChangeRecord.cs b/src/StellaOps.Feedser.Source.Distro.Suse/Internal/SuseChangeRecord.cs new file mode 100644 index 00000000..99c8833f --- /dev/null +++ b/src/StellaOps.Feedser.Source.Distro.Suse/Internal/SuseChangeRecord.cs @@ -0,0 +1,5 @@ +using System; + +namespace StellaOps.Feedser.Source.Distro.Suse.Internal; + +internal sealed record SuseChangeRecord(string FileName, DateTimeOffset ModifiedAt); diff --git a/src/StellaOps.Feedser.Source.Distro.Suse/Internal/SuseChangesParser.cs b/src/StellaOps.Feedser.Source.Distro.Suse/Internal/SuseChangesParser.cs new file mode 100644 index 00000000..4c61b48b --- /dev/null +++ b/src/StellaOps.Feedser.Source.Distro.Suse/Internal/SuseChangesParser.cs @@ -0,0 +1,81 @@ +using System; +using System.Collections.Generic; +using System.Globalization; +using System.IO; + +namespace StellaOps.Feedser.Source.Distro.Suse.Internal; + +internal static class SuseChangesParser +{ + public static IReadOnlyList<SuseChangeRecord> Parse(string csv) + { + if (string.IsNullOrWhiteSpace(csv)) + { + return Array.Empty<SuseChangeRecord>(); + } + + var records = new List<SuseChangeRecord>(); + using var reader = new StringReader(csv); + string? line; + while ((line = reader.ReadLine()) is not null) + { + if (string.IsNullOrWhiteSpace(line)) + { + continue; + } + + var parts = SplitCsvLine(line); + if (parts.Length < 2) + { + continue; + } + + var fileName = parts[0].Trim(); + if (string.IsNullOrWhiteSpace(fileName)) + { + continue; + } + + if (!DateTimeOffset.TryParse(parts[1], CultureInfo.InvariantCulture, DateTimeStyles.AssumeUniversal, out var modifiedAt)) + { + continue; + } + + records.Add(new SuseChangeRecord(fileName, modifiedAt.ToUniversalTime())); + } + + return records; + } + + private static string[] SplitCsvLine(string line) + { + var values = new List<string>(2); + var current = string.Empty; + var insideQuotes = false; + + foreach (var ch in line) + { + if (ch == '"') + { + insideQuotes = !insideQuotes; + continue; + } + + if (ch == ',' && !insideQuotes) + { + values.Add(current); + current = string.Empty; + continue; + } + + current += ch; + } + + if (!string.IsNullOrEmpty(current)) + { + values.Add(current); + } + + return values.ToArray(); + } +} diff --git a/src/StellaOps.Feedser.Source.Distro.Suse/Internal/SuseCsafParser.cs b/src/StellaOps.Feedser.Source.Distro.Suse/Internal/SuseCsafParser.cs new file mode 100644 index 00000000..3aebaec6 --- /dev/null +++ b/src/StellaOps.Feedser.Source.Distro.Suse/Internal/SuseCsafParser.cs @@ -0,0 +1,422 @@ +using System; +using System.Buffers.Text; +using System.Collections.Generic; +using System.Globalization; +using System.Text.Json; +using StellaOps.Feedser.Normalization.Distro; + +namespace StellaOps.Feedser.Source.Distro.Suse.Internal; + +internal static class SuseCsafParser +{ + public static SuseAdvisoryDto Parse(string json) + { + ArgumentException.ThrowIfNullOrEmpty(json); + + using var document = JsonDocument.Parse(json); + var root = document.RootElement; + + if (!root.TryGetProperty("document", out var documentElement)) + { + throw new InvalidOperationException("CSAF payload missing 'document' element."); + } + + var trackingElement = documentElement.GetProperty("tracking"); + var advisoryId = trackingElement.TryGetProperty("id", out var idElement) + ? idElement.GetString() + : null; + if (string.IsNullOrWhiteSpace(advisoryId)) + { + throw new InvalidOperationException("CSAF payload missing tracking.id."); + } + + var title = documentElement.TryGetProperty("title", out var titleElement) + ? titleElement.GetString() + : advisoryId; + + var summary = ExtractSummary(documentElement); + var published = ParseDate(trackingElement, "initial_release_date") + ?? ParseDate(trackingElement, "current_release_date") + ?? DateTimeOffset.UtcNow; + + var references = new List<SuseReferenceDto>(); + if (documentElement.TryGetProperty("references", out var referencesElement) && + referencesElement.ValueKind == JsonValueKind.Array) + { + foreach (var referenceElement in referencesElement.EnumerateArray()) + { + var url = referenceElement.TryGetProperty("url", out var urlElement) + ? urlElement.GetString() + : null; + + if (string.IsNullOrWhiteSpace(url)) + { + continue; + } + + references.Add(new SuseReferenceDto( + url.Trim(), + referenceElement.TryGetProperty("category", out var categoryElement) ? categoryElement.GetString() : null, + referenceElement.TryGetProperty("summary", out var summaryElement) ? summaryElement.GetString() : null)); + } + } + + var productLookup = BuildProductLookup(root); + var packageBuilders = new Dictionary<string, PackageStateBuilder>(StringComparer.OrdinalIgnoreCase); + var cveIds = new HashSet<string>(StringComparer.OrdinalIgnoreCase); + + if (root.TryGetProperty("vulnerabilities", out var vulnerabilitiesElement) && + vulnerabilitiesElement.ValueKind == JsonValueKind.Array) + { + foreach (var vulnerability in vulnerabilitiesElement.EnumerateArray()) + { + if (vulnerability.TryGetProperty("cve", out var cveElement)) + { + var cve = cveElement.GetString(); + if (!string.IsNullOrWhiteSpace(cve)) + { + cveIds.Add(cve.Trim()); + } + } + + if (vulnerability.TryGetProperty("references", out var vulnReferences) && + vulnReferences.ValueKind == JsonValueKind.Array) + { + foreach (var referenceElement in vulnReferences.EnumerateArray()) + { + var url = referenceElement.TryGetProperty("url", out var urlElement) + ? urlElement.GetString() + : null; + if (string.IsNullOrWhiteSpace(url)) + { + continue; + } + + references.Add(new SuseReferenceDto( + url.Trim(), + referenceElement.TryGetProperty("category", out var categoryElement) ? categoryElement.GetString() : null, + referenceElement.TryGetProperty("summary", out var summaryElement) ? summaryElement.GetString() : null)); + } + } + + if (!vulnerability.TryGetProperty("product_status", out var statusElement) || + statusElement.ValueKind != JsonValueKind.Object) + { + continue; + } + + foreach (var property in statusElement.EnumerateObject()) + { + var category = property.Name; + var idArray = property.Value; + if (idArray.ValueKind != JsonValueKind.Array) + { + continue; + } + + foreach (var productIdElement in idArray.EnumerateArray()) + { + var productId = productIdElement.GetString(); + if (string.IsNullOrWhiteSpace(productId)) + { + continue; + } + + if (!productLookup.TryGetValue(productId, out var product)) + { + continue; + } + + if (!packageBuilders.TryGetValue(productId, out var builder)) + { + builder = new PackageStateBuilder(product); + packageBuilders[productId] = builder; + } + + builder.ApplyStatus(category, product); + } + } + } + } + + var packages = new List<SusePackageStateDto>(packageBuilders.Count); + foreach (var builder in packageBuilders.Values) + { + if (builder.ShouldEmit) + { + packages.Add(builder.ToDto()); + } + } + + packages.Sort(static (left, right) => + { + var compare = string.Compare(left.Platform, right.Platform, StringComparison.OrdinalIgnoreCase); + if (compare != 0) + { + return compare; + } + + compare = string.Compare(left.Package, right.Package, StringComparison.OrdinalIgnoreCase); + if (compare != 0) + { + return compare; + } + + return string.Compare(left.Architecture, right.Architecture, StringComparison.OrdinalIgnoreCase); + }); + + var cveList = cveIds.Count == 0 + ? Array.Empty<string>() + : cveIds.OrderBy(static cve => cve, StringComparer.OrdinalIgnoreCase).ToArray(); + + return new SuseAdvisoryDto( + advisoryId.Trim(), + string.IsNullOrWhiteSpace(title) ? advisoryId : title!, + summary, + published, + cveList, + packages, + references); + } + + private static string? ExtractSummary(JsonElement documentElement) + { + if (!documentElement.TryGetProperty("notes", out var notesElement) || notesElement.ValueKind != JsonValueKind.Array) + { + return null; + } + + foreach (var note in notesElement.EnumerateArray()) + { + var category = note.TryGetProperty("category", out var categoryElement) + ? categoryElement.GetString() + : null; + + if (string.Equals(category, "summary", StringComparison.OrdinalIgnoreCase) + || string.Equals(category, "description", StringComparison.OrdinalIgnoreCase)) + { + return note.TryGetProperty("text", out var textElement) ? textElement.GetString() : null; + } + } + + return null; + } + + private static DateTimeOffset? ParseDate(JsonElement element, string propertyName) + { + if (!element.TryGetProperty(propertyName, out var dateElement)) + { + return null; + } + + if (dateElement.ValueKind == JsonValueKind.String && + DateTimeOffset.TryParse(dateElement.GetString(), CultureInfo.InvariantCulture, DateTimeStyles.AssumeUniversal, out var parsed)) + { + return parsed.ToUniversalTime(); + } + + return null; + } + + private static Dictionary<string, SuseProduct> BuildProductLookup(JsonElement root) + { + var lookup = new Dictionary<string, SuseProduct>(StringComparer.OrdinalIgnoreCase); + + if (!root.TryGetProperty("product_tree", out var productTree)) + { + return lookup; + } + + if (productTree.TryGetProperty("branches", out var branches) && branches.ValueKind == JsonValueKind.Array) + { + TraverseBranches(branches, null, null, lookup); + } + + return lookup; + } + + private static void TraverseBranches(JsonElement branches, string? platform, string? architecture, IDictionary<string, SuseProduct> lookup) + { + foreach (var branch in branches.EnumerateArray()) + { + var category = branch.TryGetProperty("category", out var categoryElement) + ? categoryElement.GetString() + : null; + + var name = branch.TryGetProperty("name", out var nameElement) + ? nameElement.GetString() + : null; + + var nextPlatform = platform; + var nextArchitecture = architecture; + + if (string.Equals(category, "product_family", StringComparison.OrdinalIgnoreCase) || + string.Equals(category, "product_name", StringComparison.OrdinalIgnoreCase) || + string.Equals(category, "product_version", StringComparison.OrdinalIgnoreCase)) + { + if (!string.IsNullOrWhiteSpace(name)) + { + nextPlatform = name; + } + } + + if (string.Equals(category, "architecture", StringComparison.OrdinalIgnoreCase)) + { + nextArchitecture = string.IsNullOrWhiteSpace(name) ? null : name; + } + + if (branch.TryGetProperty("product", out var productElement) && productElement.ValueKind == JsonValueKind.Object) + { + var productId = productElement.TryGetProperty("product_id", out var idElement) + ? idElement.GetString() + : null; + + if (!string.IsNullOrWhiteSpace(productId)) + { + var productName = productElement.TryGetProperty("name", out var productNameElement) + ? productNameElement.GetString() + : productId; + + var (platformName, packageSegment) = SplitProductId(productId!, nextPlatform); + if (string.IsNullOrWhiteSpace(packageSegment)) + { + packageSegment = productName; + } + + if (string.IsNullOrWhiteSpace(packageSegment)) + { + continue; + } + + if (!Nevra.TryParse(packageSegment, out var nevra) && !Nevra.TryParse(productName ?? packageSegment, out nevra)) + { + continue; + } + + lookup[productId!] = new SuseProduct( + productId!, + platformName ?? "SUSE", + nevra!, + nextArchitecture ?? nevra!.Architecture); + } + } + + if (branch.TryGetProperty("branches", out var childBranches) && childBranches.ValueKind == JsonValueKind.Array) + { + TraverseBranches(childBranches, nextPlatform, nextArchitecture, lookup); + } + } + } + + private static (string? Platform, string? Package) SplitProductId(string productId, string? currentPlatform) + { + var separatorIndex = productId.IndexOf(':'); + if (separatorIndex < 0) + { + return (currentPlatform, productId); + } + + var platform = productId[..separatorIndex]; + var package = separatorIndex < productId.Length - 1 ? productId[(separatorIndex + 1)..] : string.Empty; + var platformNormalized = string.IsNullOrWhiteSpace(platform) ? currentPlatform : platform; + var packageNormalized = string.IsNullOrWhiteSpace(package) ? null : package; + return (platformNormalized, packageNormalized); + } + + private static string FormatNevraVersion(Nevra nevra) + { + var epochSegment = nevra.HasExplicitEpoch || nevra.Epoch > 0 ? $"{nevra.Epoch}:" : string.Empty; + return $"{epochSegment}{nevra.Version}-{nevra.Release}"; + } + + private sealed record SuseProduct(string ProductId, string Platform, Nevra Nevra, string? Architecture) + { + public string Package => Nevra.Name; + + public string Version => FormatNevraVersion(Nevra); + + public string CanonicalNevra => Nevra.ToCanonicalString(); + } + + private sealed class PackageStateBuilder + { + private readonly SuseProduct _product; + + public PackageStateBuilder(SuseProduct product) + { + _product = product; + Status = null; + } + + public string Package => _product.Package; + public string Platform => _product.Platform; + public string? Architecture => _product.Architecture; + public string? IntroducedVersion { get; private set; } + public string? FixedVersion { get; private set; } + public string? LastAffectedVersion { get; private set; } + public string? Status { get; private set; } + + public bool ShouldEmit => !string.IsNullOrWhiteSpace(Status) && !string.Equals(Status, "not_affected", StringComparison.OrdinalIgnoreCase); + + public void ApplyStatus(string category, SuseProduct product) + { + if (string.IsNullOrWhiteSpace(category)) + { + return; + } + + switch (category.ToLowerInvariant()) + { + case "recommended": + case "fixed": + FixedVersion = product.Version; + Status = "resolved"; + break; + + case "known_affected": + case "known_vulnerable": + LastAffectedVersion = product.Version; + Status ??= "open"; + break; + + case "first_affected": + IntroducedVersion ??= product.Version; + Status ??= "open"; + break; + + case "under_investigation": + Status ??= "investigating"; + break; + + case "known_not_affected": + Status = "not_affected"; + IntroducedVersion = null; + FixedVersion = null; + LastAffectedVersion = null; + break; + } + } + + public SusePackageStateDto ToDto() + { + var status = Status ?? "unknown"; + var introduced = IntroducedVersion; + var lastAffected = LastAffectedVersion; + + if (string.Equals(status, "resolved", StringComparison.OrdinalIgnoreCase) && string.IsNullOrWhiteSpace(FixedVersion)) + { + status = "open"; + } + + return new SusePackageStateDto( + Package, + Platform, + Architecture, + _product.CanonicalNevra, + introduced, + FixedVersion, + lastAffected, + status); + } + } +} diff --git a/src/StellaOps.Feedser.Source.Distro.Suse/Internal/SuseCursor.cs b/src/StellaOps.Feedser.Source.Distro.Suse/Internal/SuseCursor.cs new file mode 100644 index 00000000..8ac220ad --- /dev/null +++ b/src/StellaOps.Feedser.Source.Distro.Suse/Internal/SuseCursor.cs @@ -0,0 +1,177 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using MongoDB.Bson; + +namespace StellaOps.Feedser.Source.Distro.Suse.Internal; + +internal sealed record SuseCursor( + DateTimeOffset? LastModified, + IReadOnlyCollection<string> ProcessedIds, + IReadOnlyCollection<Guid> PendingDocuments, + IReadOnlyCollection<Guid> PendingMappings, + IReadOnlyDictionary<string, SuseFetchCacheEntry> FetchCache) +{ + private static readonly IReadOnlyCollection<string> EmptyStringList = Array.Empty<string>(); + private static readonly IReadOnlyCollection<Guid> EmptyGuidList = Array.Empty<Guid>(); + private static readonly IReadOnlyDictionary<string, SuseFetchCacheEntry> EmptyCache = + new Dictionary<string, SuseFetchCacheEntry>(StringComparer.OrdinalIgnoreCase); + + public static SuseCursor Empty { get; } = new(null, EmptyStringList, EmptyGuidList, EmptyGuidList, EmptyCache); + + public static SuseCursor FromBson(BsonDocument? document) + { + if (document is null || document.ElementCount == 0) + { + return Empty; + } + + DateTimeOffset? lastModified = null; + if (document.TryGetValue("lastModified", out var lastValue)) + { + lastModified = lastValue.BsonType switch + { + BsonType.DateTime => DateTime.SpecifyKind(lastValue.ToUniversalTime(), DateTimeKind.Utc), + BsonType.String when DateTimeOffset.TryParse(lastValue.AsString, out var parsed) => parsed.ToUniversalTime(), + _ => null, + }; + } + + var processed = ReadStringSet(document, "processedIds"); + var pendingDocs = ReadGuidSet(document, "pendingDocuments"); + var pendingMappings = ReadGuidSet(document, "pendingMappings"); + var cache = ReadCache(document); + + return new SuseCursor(lastModified, processed, pendingDocs, pendingMappings, cache); + } + + public BsonDocument ToBsonDocument() + { + var document = new BsonDocument + { + ["pendingDocuments"] = new BsonArray(PendingDocuments.Select(static id => id.ToString())), + ["pendingMappings"] = new BsonArray(PendingMappings.Select(static id => id.ToString())), + }; + + if (LastModified.HasValue) + { + document["lastModified"] = LastModified.Value.UtcDateTime; + } + + if (ProcessedIds.Count > 0) + { + document["processedIds"] = new BsonArray(ProcessedIds); + } + + if (FetchCache.Count > 0) + { + var cacheDocument = new BsonDocument(); + foreach (var (key, entry) in FetchCache) + { + cacheDocument[key] = entry.ToBsonDocument(); + } + + document["fetchCache"] = cacheDocument; + } + + return document; + } + + public SuseCursor WithPendingDocuments(IEnumerable<Guid> ids) + => this with { PendingDocuments = ids?.Distinct().ToArray() ?? EmptyGuidList }; + + public SuseCursor WithPendingMappings(IEnumerable<Guid> ids) + => this with { PendingMappings = ids?.Distinct().ToArray() ?? EmptyGuidList }; + + public SuseCursor WithFetchCache(IDictionary<string, SuseFetchCacheEntry>? cache) + { + if (cache is null || cache.Count == 0) + { + return this with { FetchCache = EmptyCache }; + } + + return this with { FetchCache = new Dictionary<string, SuseFetchCacheEntry>(cache, StringComparer.OrdinalIgnoreCase) }; + } + + public SuseCursor WithProcessed(DateTimeOffset modified, IEnumerable<string> ids) + => this with + { + LastModified = modified.ToUniversalTime(), + ProcessedIds = ids?.Where(static id => !string.IsNullOrWhiteSpace(id)) + .Select(static id => id.Trim()) + .Distinct(StringComparer.OrdinalIgnoreCase) + .ToArray() ?? EmptyStringList + }; + + public bool TryGetCache(string key, out SuseFetchCacheEntry entry) + { + if (FetchCache.Count == 0) + { + entry = SuseFetchCacheEntry.Empty; + return false; + } + + return FetchCache.TryGetValue(key, out entry!); + } + + private static IReadOnlyCollection<string> ReadStringSet(BsonDocument document, string field) + { + if (!document.TryGetValue(field, out var value) || value is not BsonArray array) + { + return EmptyStringList; + } + + var list = new List<string>(array.Count); + foreach (var element in array) + { + if (element.BsonType == BsonType.String) + { + var str = element.AsString.Trim(); + if (!string.IsNullOrWhiteSpace(str)) + { + list.Add(str); + } + } + } + + return list; + } + + private static IReadOnlyCollection<Guid> ReadGuidSet(BsonDocument document, string field) + { + if (!document.TryGetValue(field, out var value) || value is not BsonArray array) + { + return EmptyGuidList; + } + + var list = new List<Guid>(array.Count); + foreach (var element in array) + { + if (Guid.TryParse(element.ToString(), out var guid)) + { + list.Add(guid); + } + } + + return list; + } + + private static IReadOnlyDictionary<string, SuseFetchCacheEntry> ReadCache(BsonDocument document) + { + if (!document.TryGetValue("fetchCache", out var value) || value is not BsonDocument cacheDocument || cacheDocument.ElementCount == 0) + { + return EmptyCache; + } + + var cache = new Dictionary<string, SuseFetchCacheEntry>(StringComparer.OrdinalIgnoreCase); + foreach (var element in cacheDocument.Elements) + { + if (element.Value is BsonDocument entry) + { + cache[element.Name] = SuseFetchCacheEntry.FromBson(entry); + } + } + + return cache; + } +} diff --git a/src/StellaOps.Feedser.Source.Distro.Suse/Internal/SuseFetchCacheEntry.cs b/src/StellaOps.Feedser.Source.Distro.Suse/Internal/SuseFetchCacheEntry.cs new file mode 100644 index 00000000..2d70dcb5 --- /dev/null +++ b/src/StellaOps.Feedser.Source.Distro.Suse/Internal/SuseFetchCacheEntry.cs @@ -0,0 +1,76 @@ +using System; +using MongoDB.Bson; + +namespace StellaOps.Feedser.Source.Distro.Suse.Internal; + +internal sealed record SuseFetchCacheEntry(string? ETag, DateTimeOffset? LastModified) +{ + public static SuseFetchCacheEntry Empty { get; } = new(null, null); + + public static SuseFetchCacheEntry FromDocument(StellaOps.Feedser.Storage.Mongo.Documents.DocumentRecord document) + => new(document.Etag, document.LastModified); + + public static SuseFetchCacheEntry FromBson(BsonDocument document) + { + if (document is null || document.ElementCount == 0) + { + return Empty; + } + + string? etag = null; + DateTimeOffset? lastModified = null; + + if (document.TryGetValue("etag", out var etagValue) && etagValue.BsonType == BsonType.String) + { + etag = etagValue.AsString; + } + + if (document.TryGetValue("lastModified", out var modifiedValue)) + { + lastModified = modifiedValue.BsonType switch + { + BsonType.DateTime => DateTime.SpecifyKind(modifiedValue.ToUniversalTime(), DateTimeKind.Utc), + BsonType.String when DateTimeOffset.TryParse(modifiedValue.AsString, out var parsed) => parsed.ToUniversalTime(), + _ => null, + }; + } + + return new SuseFetchCacheEntry(etag, lastModified); + } + + public BsonDocument ToBsonDocument() + { + var document = new BsonDocument(); + if (!string.IsNullOrWhiteSpace(ETag)) + { + document["etag"] = ETag; + } + + if (LastModified.HasValue) + { + document["lastModified"] = LastModified.Value.UtcDateTime; + } + + return document; + } + + public bool Matches(StellaOps.Feedser.Storage.Mongo.Documents.DocumentRecord document) + { + if (document is null) + { + return false; + } + + if (!string.Equals(ETag, document.Etag, StringComparison.Ordinal)) + { + return false; + } + + if (LastModified.HasValue && document.LastModified.HasValue) + { + return LastModified.Value.UtcDateTime == document.LastModified.Value.UtcDateTime; + } + + return !LastModified.HasValue && !document.LastModified.HasValue; + } +} diff --git a/src/StellaOps.Feedser.Source.Distro.Suse/Internal/SuseMapper.cs b/src/StellaOps.Feedser.Source.Distro.Suse/Internal/SuseMapper.cs new file mode 100644 index 00000000..6e22586b --- /dev/null +++ b/src/StellaOps.Feedser.Source.Distro.Suse/Internal/SuseMapper.cs @@ -0,0 +1,313 @@ +using System; +using System.Collections.Generic; +using System.Globalization; +using System.Linq; +using StellaOps.Feedser.Models; +using StellaOps.Feedser.Normalization.Distro; +using StellaOps.Feedser.Storage.Mongo.Documents; + +namespace StellaOps.Feedser.Source.Distro.Suse.Internal; + +internal static class SuseMapper +{ + public static Advisory Map(SuseAdvisoryDto dto, DocumentRecord document, DateTimeOffset recordedAt) + { + ArgumentNullException.ThrowIfNull(dto); + ArgumentNullException.ThrowIfNull(document); + + var aliases = BuildAliases(dto); + var references = BuildReferences(dto, recordedAt); + var packages = BuildPackages(dto, recordedAt); + + var fetchProvenance = new AdvisoryProvenance( + SuseConnectorPlugin.SourceName, + "document", + document.Uri, + document.FetchedAt.ToUniversalTime()); + + var mapProvenance = new AdvisoryProvenance( + SuseConnectorPlugin.SourceName, + "mapping", + dto.AdvisoryId, + recordedAt); + + var published = dto.Published; + var modified = DateTimeOffset.Compare(recordedAt, dto.Published) >= 0 ? recordedAt : dto.Published; + + return new Advisory( + advisoryKey: dto.AdvisoryId, + title: dto.Title ?? dto.AdvisoryId, + summary: dto.Summary, + language: "en", + published: published, + modified: modified, + severity: null, + exploitKnown: false, + aliases: aliases, + references: references, + affectedPackages: packages, + cvssMetrics: Array.Empty<CvssMetric>(), + provenance: new[] { fetchProvenance, mapProvenance }); + } + + private static string[] BuildAliases(SuseAdvisoryDto dto) + { + var aliases = new HashSet<string>(StringComparer.OrdinalIgnoreCase) + { + dto.AdvisoryId + }; + + foreach (var cve in dto.CveIds ?? Array.Empty<string>()) + { + if (!string.IsNullOrWhiteSpace(cve)) + { + aliases.Add(cve.Trim()); + } + } + + return aliases.OrderBy(static alias => alias, StringComparer.OrdinalIgnoreCase).ToArray(); + } + + private static AdvisoryReference[] BuildReferences(SuseAdvisoryDto dto, DateTimeOffset recordedAt) + { + if (dto.References is null || dto.References.Count == 0) + { + return Array.Empty<AdvisoryReference>(); + } + + var references = new List<AdvisoryReference>(dto.References.Count); + foreach (var reference in dto.References) + { + if (string.IsNullOrWhiteSpace(reference.Url)) + { + continue; + } + + try + { + var provenance = new AdvisoryProvenance( + SuseConnectorPlugin.SourceName, + "reference", + reference.Url, + recordedAt); + + references.Add(new AdvisoryReference( + reference.Url.Trim(), + NormalizeReferenceKind(reference.Kind), + reference.Kind, + reference.Title, + provenance)); + } + catch (ArgumentException) + { + // Ignore malformed URLs to keep advisory mapping resilient. + } + } + + return references.Count == 0 + ? Array.Empty<AdvisoryReference>() + : references + .OrderBy(static reference => reference.Url, StringComparer.OrdinalIgnoreCase) + .ToArray(); + } + + private static string? NormalizeReferenceKind(string? kind) + { + if (string.IsNullOrWhiteSpace(kind)) + { + return null; + } + + return kind.Trim().ToLowerInvariant() switch + { + "cve" => "cve", + "self" => "advisory", + "external" => "external", + _ => null, + }; + } + + private static IReadOnlyList<AffectedPackage> BuildPackages(SuseAdvisoryDto dto, DateTimeOffset recordedAt) + { + if (dto.Packages is null || dto.Packages.Count == 0) + { + return Array.Empty<AffectedPackage>(); + } + + var packages = new List<AffectedPackage>(dto.Packages.Count); + foreach (var package in dto.Packages) + { + if (string.IsNullOrWhiteSpace(package.CanonicalNevra)) + { + continue; + } + + Nevra? nevra; + if (!Nevra.TryParse(package.CanonicalNevra, out nevra)) + { + continue; + } + + var affectedProvenance = new AdvisoryProvenance( + SuseConnectorPlugin.SourceName, + "affected", + $"{package.Platform}:{package.CanonicalNevra}", + recordedAt); + + var ranges = BuildVersionRanges(package, nevra!, recordedAt); + if (ranges.Count == 0 && string.Equals(package.Status, "not_affected", StringComparison.OrdinalIgnoreCase)) + { + continue; + } + + packages.Add(new AffectedPackage( + AffectedPackageTypes.Rpm, + identifier: nevra!.ToCanonicalString(), + platform: package.Platform, + versionRanges: ranges, + statuses: BuildStatuses(package, affectedProvenance), + provenance: new[] { affectedProvenance })); + } + + return packages.Count == 0 + ? Array.Empty<AffectedPackage>() + : packages + .OrderBy(static pkg => pkg.Platform, StringComparer.OrdinalIgnoreCase) + .ThenBy(static pkg => pkg.Identifier, StringComparer.OrdinalIgnoreCase) + .ToArray(); + } + + private static IReadOnlyList<AffectedPackageStatus> BuildStatuses(SusePackageStateDto package, AdvisoryProvenance provenance) + { + if (string.IsNullOrWhiteSpace(package.Status)) + { + return Array.Empty<AffectedPackageStatus>(); + } + + return new[] + { + new AffectedPackageStatus(package.Status, provenance) + }; + } + + private static IReadOnlyList<AffectedVersionRange> BuildVersionRanges(SusePackageStateDto package, Nevra nevra, DateTimeOffset recordedAt) + { + var introducedComponent = ParseNevraComponent(package.IntroducedVersion, nevra); + var fixedComponent = ParseNevraComponent(package.FixedVersion, nevra); + var lastAffectedComponent = ParseNevraComponent(package.LastAffectedVersion, nevra); + + if (introducedComponent is null && fixedComponent is null && lastAffectedComponent is null) + { + return Array.Empty<AffectedVersionRange>(); + } + + var rangeProvenance = new AdvisoryProvenance( + SuseConnectorPlugin.SourceName, + "range", + $"{package.Platform}:{nevra.ToCanonicalString()}", + recordedAt); + + var extensions = new Dictionary<string, string>(StringComparer.Ordinal) + { + ["suse.status"] = package.Status + }; + + var rangeExpression = BuildRangeExpression(package.IntroducedVersion, package.FixedVersion, package.LastAffectedVersion); + + var range = new AffectedVersionRange( + rangeKind: "nevra", + introducedVersion: package.IntroducedVersion, + fixedVersion: package.FixedVersion, + lastAffectedVersion: package.LastAffectedVersion, + rangeExpression: rangeExpression, + provenance: rangeProvenance, + primitives: new RangePrimitives( + SemVer: null, + Nevra: new NevraPrimitive(introducedComponent, fixedComponent, lastAffectedComponent), + Evr: null, + VendorExtensions: extensions)); + + return new[] { range }; + } + + private static NevraComponent? ParseNevraComponent(string? version, Nevra nevra) + { + if (string.IsNullOrWhiteSpace(version)) + { + return null; + } + + if (!TrySplitNevraVersion(version.Trim(), out var epoch, out var ver, out var rel)) + { + return null; + } + + return new NevraComponent( + nevra.Name, + epoch, + ver, + rel, + string.IsNullOrWhiteSpace(nevra.Architecture) ? null : nevra.Architecture); + } + + private static bool TrySplitNevraVersion(string value, out int epoch, out string version, out string release) + { + epoch = 0; + version = string.Empty; + release = string.Empty; + + if (string.IsNullOrWhiteSpace(value)) + { + return false; + } + + var trimmed = value.Trim(); + var dashIndex = trimmed.LastIndexOf('-'); + if (dashIndex <= 0 || dashIndex >= trimmed.Length - 1) + { + return false; + } + + release = trimmed[(dashIndex + 1)..]; + var versionSegment = trimmed[..dashIndex]; + + var epochIndex = versionSegment.IndexOf(':'); + if (epochIndex >= 0) + { + var epochPart = versionSegment[..epochIndex]; + version = epochIndex < versionSegment.Length - 1 ? versionSegment[(epochIndex + 1)..] : string.Empty; + if (epochPart.Length > 0 && !int.TryParse(epochPart, NumberStyles.Integer, CultureInfo.InvariantCulture, out epoch)) + { + epoch = 0; + return false; + } + } + else + { + version = versionSegment; + } + + return !string.IsNullOrWhiteSpace(version) && !string.IsNullOrWhiteSpace(release); + } + + private static string? BuildRangeExpression(string? introduced, string? fixedVersion, string? lastAffected) + { + var parts = new List<string>(3); + if (!string.IsNullOrWhiteSpace(introduced)) + { + parts.Add($"introduced:{introduced}"); + } + + if (!string.IsNullOrWhiteSpace(fixedVersion)) + { + parts.Add($"fixed:{fixedVersion}"); + } + + if (!string.IsNullOrWhiteSpace(lastAffected)) + { + parts.Add($"last:{lastAffected}"); + } + + return parts.Count == 0 ? null : string.Join(" ", parts); + } +} diff --git a/src/StellaOps.Feedser.Source.Distro.Suse/Jobs.cs b/src/StellaOps.Feedser.Source.Distro.Suse/Jobs.cs new file mode 100644 index 00000000..d2fcd2fa --- /dev/null +++ b/src/StellaOps.Feedser.Source.Distro.Suse/Jobs.cs @@ -0,0 +1,46 @@ +using System; +using System.Threading; +using System.Threading.Tasks; +using StellaOps.Feedser.Core.Jobs; + +namespace StellaOps.Feedser.Source.Distro.Suse; + +internal static class SuseJobKinds +{ + public const string Fetch = "source:suse:fetch"; + public const string Parse = "source:suse:parse"; + public const string Map = "source:suse:map"; +} + +internal sealed class SuseFetchJob : IJob +{ + private readonly SuseConnector _connector; + + public SuseFetchJob(SuseConnector connector) + => _connector = connector ?? throw new ArgumentNullException(nameof(connector)); + + public Task ExecuteAsync(JobExecutionContext context, CancellationToken cancellationToken) + => _connector.FetchAsync(context.Services, cancellationToken); +} + +internal sealed class SuseParseJob : IJob +{ + private readonly SuseConnector _connector; + + public SuseParseJob(SuseConnector connector) + => _connector = connector ?? throw new ArgumentNullException(nameof(connector)); + + public Task ExecuteAsync(JobExecutionContext context, CancellationToken cancellationToken) + => _connector.ParseAsync(context.Services, cancellationToken); +} + +internal sealed class SuseMapJob : IJob +{ + private readonly SuseConnector _connector; + + public SuseMapJob(SuseConnector connector) + => _connector = connector ?? throw new ArgumentNullException(nameof(connector)); + + public Task ExecuteAsync(JobExecutionContext context, CancellationToken cancellationToken) + => _connector.MapAsync(context.Services, cancellationToken); +} diff --git a/src/StellaOps.Feedser.Source.Distro.Suse/StellaOps.Feedser.Source.Distro.Suse.csproj b/src/StellaOps.Feedser.Source.Distro.Suse/StellaOps.Feedser.Source.Distro.Suse.csproj new file mode 100644 index 00000000..96165c66 --- /dev/null +++ b/src/StellaOps.Feedser.Source.Distro.Suse/StellaOps.Feedser.Source.Distro.Suse.csproj @@ -0,0 +1,17 @@ +<Project Sdk="Microsoft.NET.Sdk"> + + <PropertyGroup> + <TargetFramework>net10.0</TargetFramework> + <ImplicitUsings>enable</ImplicitUsings> + <Nullable>enable</Nullable> + </PropertyGroup> + + <ItemGroup> + <ProjectReference Include="../StellaOps.Plugin/StellaOps.Plugin.csproj" /> + + <ProjectReference Include="../StellaOps.Feedser.Source.Common/StellaOps.Feedser.Source.Common.csproj" /> + <ProjectReference Include="../StellaOps.Feedser.Models/StellaOps.Feedser.Models.csproj" /> + <ProjectReference Include="../StellaOps.Feedser.Normalization/StellaOps.Feedser.Normalization.csproj" /> + <ProjectReference Include="../StellaOps.Feedser.Storage.Mongo/StellaOps.Feedser.Storage.Mongo.csproj" /> + </ItemGroup> +</Project> diff --git a/src/StellaOps.Feedser.Source.Distro.Suse/SuseConnector.cs b/src/StellaOps.Feedser.Source.Distro.Suse/SuseConnector.cs new file mode 100644 index 00000000..c21b19d8 --- /dev/null +++ b/src/StellaOps.Feedser.Source.Distro.Suse/SuseConnector.cs @@ -0,0 +1,573 @@ +using System; +using System.Collections.Generic; +using System.Globalization; +using System.IO; +using System.Linq; +using System.Text; +using System.Text.Json; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Options; +using MongoDB.Bson; +using MongoDB.Bson.IO; +using StellaOps.Feedser.Models; +using StellaOps.Feedser.Source.Common; +using StellaOps.Feedser.Source.Common.Fetch; +using StellaOps.Feedser.Source.Distro.Suse.Configuration; +using StellaOps.Feedser.Source.Distro.Suse.Internal; +using StellaOps.Feedser.Storage.Mongo; +using StellaOps.Feedser.Storage.Mongo.Advisories; +using StellaOps.Feedser.Storage.Mongo.Documents; +using StellaOps.Feedser.Storage.Mongo.Dtos; +using StellaOps.Plugin; + +namespace StellaOps.Feedser.Source.Distro.Suse; + +public sealed class SuseConnector : IFeedConnector +{ + private static readonly Action<ILogger, string, int, Exception?> LogMapped = + LoggerMessage.Define<string, int>( + LogLevel.Information, + new EventId(1, "SuseMapped"), + "SUSE advisory {AdvisoryId} mapped with {AffectedCount} affected packages"); + + private readonly SourceFetchService _fetchService; + private readonly RawDocumentStorage _rawDocumentStorage; + private readonly IDocumentStore _documentStore; + private readonly IDtoStore _dtoStore; + private readonly IAdvisoryStore _advisoryStore; + private readonly ISourceStateRepository _stateRepository; + private readonly SuseOptions _options; + private readonly TimeProvider _timeProvider; + private readonly ILogger<SuseConnector> _logger; + + public SuseConnector( + SourceFetchService fetchService, + RawDocumentStorage rawDocumentStorage, + IDocumentStore documentStore, + IDtoStore dtoStore, + IAdvisoryStore advisoryStore, + ISourceStateRepository stateRepository, + IOptions<SuseOptions> options, + TimeProvider? timeProvider, + ILogger<SuseConnector> logger) + { + _fetchService = fetchService ?? throw new ArgumentNullException(nameof(fetchService)); + _rawDocumentStorage = rawDocumentStorage ?? throw new ArgumentNullException(nameof(rawDocumentStorage)); + _documentStore = documentStore ?? throw new ArgumentNullException(nameof(documentStore)); + _dtoStore = dtoStore ?? throw new ArgumentNullException(nameof(dtoStore)); + _advisoryStore = advisoryStore ?? throw new ArgumentNullException(nameof(advisoryStore)); + _stateRepository = stateRepository ?? throw new ArgumentNullException(nameof(stateRepository)); + _options = (options ?? throw new ArgumentNullException(nameof(options))).Value ?? throw new ArgumentNullException(nameof(options)); + _options.Validate(); + _timeProvider = timeProvider ?? TimeProvider.System; + _logger = logger ?? throw new ArgumentNullException(nameof(logger)); + } + + public string SourceName => SuseConnectorPlugin.SourceName; + + public async Task FetchAsync(IServiceProvider services, CancellationToken cancellationToken) + { + ArgumentNullException.ThrowIfNull(services); + + var cursor = await GetCursorAsync(cancellationToken).ConfigureAwait(false); + var now = _timeProvider.GetUtcNow(); + + var pendingDocuments = new HashSet<Guid>(cursor.PendingDocuments); + var pendingMappings = new HashSet<Guid>(cursor.PendingMappings); + var fetchCache = new Dictionary<string, SuseFetchCacheEntry>(cursor.FetchCache, StringComparer.OrdinalIgnoreCase); + var touchedResources = new HashSet<string>(StringComparer.OrdinalIgnoreCase); + + var changesUri = _options.ChangesEndpoint; + var changesKey = changesUri.ToString(); + touchedResources.Add(changesKey); + + cursor.TryGetCache(changesKey, out var cachedChanges); + + var changesRequest = new SourceFetchRequest(SuseOptions.HttpClientName, SourceName, changesUri) + { + Metadata = new Dictionary<string, string>(StringComparer.Ordinal) + { + ["suse.type"] = "changes" + }, + AcceptHeaders = new[] { "text/csv", "text/plain" }, + TimeoutOverride = _options.FetchTimeout, + ETag = cachedChanges?.ETag, + LastModified = cachedChanges?.LastModified, + }; + + SourceFetchResult changesResult; + try + { + changesResult = await _fetchService.FetchAsync(changesRequest, cancellationToken).ConfigureAwait(false); + } + catch (Exception ex) + { + _logger.LogError(ex, "SUSE changes.csv fetch failed from {Uri}", changesUri); + await _stateRepository.MarkFailureAsync(SourceName, now, TimeSpan.FromMinutes(5), ex.Message, cancellationToken).ConfigureAwait(false); + throw; + } + + var maxModified = cursor.LastModified ?? DateTimeOffset.MinValue; + var processedUpdated = false; + var processedIds = new HashSet<string>(cursor.ProcessedIds, StringComparer.OrdinalIgnoreCase); + var currentWindowIds = new HashSet<string>(StringComparer.OrdinalIgnoreCase); + + IReadOnlyList<SuseChangeRecord> changeRecords = Array.Empty<SuseChangeRecord>(); + if (changesResult.IsNotModified) + { + if (cursor.FetchCache.TryGetValue(changesKey, out var existingCache)) + { + fetchCache[changesKey] = existingCache; + } + } + else if (changesResult.IsSuccess && changesResult.Document is not null) + { + fetchCache[changesKey] = SuseFetchCacheEntry.FromDocument(changesResult.Document); + if (changesResult.Document.GridFsId.HasValue) + { + byte[] changesBytes; + try + { + changesBytes = await _rawDocumentStorage.DownloadAsync(changesResult.Document.GridFsId.Value, cancellationToken).ConfigureAwait(false); + } + catch (Exception ex) + { + _logger.LogError(ex, "Failed to download SUSE changes.csv document {DocumentId}", changesResult.Document.Id); + throw; + } + + var csv = Encoding.UTF8.GetString(changesBytes); + changeRecords = SuseChangesParser.Parse(csv); + } + } + + if (changeRecords.Count > 0) + { + var baseline = (cursor.LastModified ?? (now - _options.InitialBackfill)) - _options.ResumeOverlap; + if (baseline < DateTimeOffset.UnixEpoch) + { + baseline = DateTimeOffset.UnixEpoch; + } + + ProvenanceDiagnostics.ReportResumeWindow(SourceName, baseline, _logger); + + var candidates = changeRecords + .Where(record => record.ModifiedAt >= baseline) + .OrderBy(record => record.ModifiedAt) + .ThenBy(record => record.FileName, StringComparer.OrdinalIgnoreCase) + .ToList(); + + if (candidates.Count == 0) + { + candidates = changeRecords + .OrderByDescending(record => record.ModifiedAt) + .ThenBy(record => record.FileName, StringComparer.OrdinalIgnoreCase) + .Take(_options.MaxAdvisoriesPerFetch) + .OrderBy(record => record.ModifiedAt) + .ThenBy(record => record.FileName, StringComparer.OrdinalIgnoreCase) + .ToList(); + } + else if (candidates.Count > _options.MaxAdvisoriesPerFetch) + { + candidates = candidates + .OrderByDescending(record => record.ModifiedAt) + .ThenBy(record => record.FileName, StringComparer.OrdinalIgnoreCase) + .Take(_options.MaxAdvisoriesPerFetch) + .OrderBy(record => record.ModifiedAt) + .ThenBy(record => record.FileName, StringComparer.OrdinalIgnoreCase) + .ToList(); + } + + foreach (var record in candidates) + { + cancellationToken.ThrowIfCancellationRequested(); + + var detailUri = new Uri(_options.AdvisoryBaseUri, record.FileName); + var cacheKey = detailUri.AbsoluteUri; + touchedResources.Add(cacheKey); + + cursor.TryGetCache(cacheKey, out var cachedEntry); + var existing = await _documentStore.FindBySourceAndUriAsync(SourceName, cacheKey, cancellationToken).ConfigureAwait(false); + + var metadata = new Dictionary<string, string>(StringComparer.Ordinal) + { + ["suse.file"] = record.FileName, + ["suse.modified"] = record.ModifiedAt.ToString("O", CultureInfo.InvariantCulture) + }; + + if (!metadata.ContainsKey("suse.id") && existing?.Metadata?.TryGetValue("suse.id", out var existingId) == true) + { + metadata["suse.id"] = existingId; + } + + var request = new SourceFetchRequest(SuseOptions.HttpClientName, SourceName, detailUri) + { + Metadata = metadata, + AcceptHeaders = new[] { "application/json", "text/json" }, + TimeoutOverride = _options.FetchTimeout, + ETag = existing?.Etag ?? cachedEntry?.ETag, + LastModified = existing?.LastModified ?? cachedEntry?.LastModified, + }; + + SourceFetchResult result; + try + { + result = await _fetchService.FetchAsync(request, cancellationToken).ConfigureAwait(false); + } + catch (Exception ex) + { + _logger.LogError(ex, "Failed to fetch SUSE advisory {FileName}", record.FileName); + await _stateRepository.MarkFailureAsync(SourceName, now, TimeSpan.FromMinutes(5), ex.Message, cancellationToken).ConfigureAwait(false); + throw; + } + + if (result.IsNotModified) + { + if (existing is not null) + { + fetchCache[cacheKey] = SuseFetchCacheEntry.FromDocument(existing); + if (string.Equals(existing.Status, DocumentStatuses.Mapped, StringComparison.Ordinal)) + { + pendingDocuments.Remove(existing.Id); + pendingMappings.Remove(existing.Id); + } + } + + continue; + } + + if (!result.IsSuccess || result.Document is null) + { + continue; + } + + fetchCache[cacheKey] = SuseFetchCacheEntry.FromDocument(result.Document); + pendingDocuments.Add(result.Document.Id); + pendingMappings.Remove(result.Document.Id); + currentWindowIds.Add(record.FileName); + + if (record.ModifiedAt > maxModified) + { + maxModified = record.ModifiedAt; + processedUpdated = true; + } + } + } + + if (fetchCache.Count > 0 && touchedResources.Count > 0) + { + var staleKeys = fetchCache.Keys.Where(key => !touchedResources.Contains(key)).ToArray(); + foreach (var key in staleKeys) + { + fetchCache.Remove(key); + } + } + + var updatedCursor = cursor + .WithPendingDocuments(pendingDocuments) + .WithPendingMappings(pendingMappings) + .WithFetchCache(fetchCache); + + if (processedUpdated && currentWindowIds.Count > 0) + { + updatedCursor = updatedCursor.WithProcessed(maxModified, currentWindowIds); + } + + await UpdateCursorAsync(updatedCursor, cancellationToken).ConfigureAwait(false); + } + + public async Task ParseAsync(IServiceProvider services, CancellationToken cancellationToken) + { + ArgumentNullException.ThrowIfNull(services); + + var cursor = await GetCursorAsync(cancellationToken).ConfigureAwait(false); + if (cursor.PendingDocuments.Count == 0) + { + return; + } + + var remaining = cursor.PendingDocuments.ToList(); + var pendingMappings = cursor.PendingMappings.ToList(); + + foreach (var documentId in cursor.PendingDocuments) + { + cancellationToken.ThrowIfCancellationRequested(); + + var document = await _documentStore.FindAsync(documentId, cancellationToken).ConfigureAwait(false); + if (document is null) + { + remaining.Remove(documentId); + continue; + } + + if (!document.GridFsId.HasValue) + { + _logger.LogWarning("SUSE document {DocumentId} missing GridFS payload", document.Id); + await _documentStore.UpdateStatusAsync(document.Id, DocumentStatuses.Failed, cancellationToken).ConfigureAwait(false); + remaining.Remove(documentId); + continue; + } + + byte[] bytes; + try + { + bytes = await _rawDocumentStorage.DownloadAsync(document.GridFsId.Value, cancellationToken).ConfigureAwait(false); + } + catch (Exception ex) + { + _logger.LogError(ex, "Failed to download SUSE document {DocumentId}", document.Id); + throw; + } + + SuseAdvisoryDto dto; + try + { + var json = Encoding.UTF8.GetString(bytes); + dto = SuseCsafParser.Parse(json); + } + catch (Exception ex) + { + _logger.LogWarning(ex, "Failed to parse SUSE advisory {Uri}", document.Uri); + await _documentStore.UpdateStatusAsync(document.Id, DocumentStatuses.Failed, cancellationToken).ConfigureAwait(false); + remaining.Remove(documentId); + continue; + } + + var metadata = document.Metadata is null + ? new Dictionary<string, string>(StringComparer.Ordinal) + : new Dictionary<string, string>(document.Metadata, StringComparer.Ordinal); + + metadata["suse.id"] = dto.AdvisoryId; + var updatedDocument = document with { Metadata = metadata }; + await _documentStore.UpsertAsync(updatedDocument, cancellationToken).ConfigureAwait(false); + + var payload = ToBson(dto); + var dtoRecord = new DtoRecord(Guid.NewGuid(), document.Id, SourceName, "suse.csaf.v1", payload, _timeProvider.GetUtcNow()); + + await _dtoStore.UpsertAsync(dtoRecord, cancellationToken).ConfigureAwait(false); + await _documentStore.UpdateStatusAsync(document.Id, DocumentStatuses.PendingMap, cancellationToken).ConfigureAwait(false); + + remaining.Remove(documentId); + if (!pendingMappings.Contains(documentId)) + { + pendingMappings.Add(documentId); + } + } + + var updatedCursor = cursor + .WithPendingDocuments(remaining) + .WithPendingMappings(pendingMappings); + + await UpdateCursorAsync(updatedCursor, cancellationToken).ConfigureAwait(false); + } + + public async Task MapAsync(IServiceProvider services, CancellationToken cancellationToken) + { + ArgumentNullException.ThrowIfNull(services); + + var cursor = await GetCursorAsync(cancellationToken).ConfigureAwait(false); + if (cursor.PendingMappings.Count == 0) + { + return; + } + + var pendingMappings = cursor.PendingMappings.ToList(); + + foreach (var documentId in cursor.PendingMappings) + { + cancellationToken.ThrowIfCancellationRequested(); + + var dtoRecord = await _dtoStore.FindByDocumentIdAsync(documentId, cancellationToken).ConfigureAwait(false); + var document = await _documentStore.FindAsync(documentId, cancellationToken).ConfigureAwait(false); + if (dtoRecord is null || document is null) + { + pendingMappings.Remove(documentId); + continue; + } + + SuseAdvisoryDto dto; + try + { + dto = FromBson(dtoRecord.Payload); + } + catch (Exception ex) + { + _logger.LogError(ex, "Failed to deserialize SUSE DTO for document {DocumentId}", documentId); + await _documentStore.UpdateStatusAsync(documentId, DocumentStatuses.Failed, cancellationToken).ConfigureAwait(false); + pendingMappings.Remove(documentId); + continue; + } + + var advisory = SuseMapper.Map(dto, document, _timeProvider.GetUtcNow()); + await _advisoryStore.UpsertAsync(advisory, cancellationToken).ConfigureAwait(false); + await _documentStore.UpdateStatusAsync(documentId, DocumentStatuses.Mapped, cancellationToken).ConfigureAwait(false); + pendingMappings.Remove(documentId); + + LogMapped(_logger, dto.AdvisoryId, advisory.AffectedPackages.Length, null); + } + + var updatedCursor = cursor.WithPendingMappings(pendingMappings); + await UpdateCursorAsync(updatedCursor, cancellationToken).ConfigureAwait(false); + } + + private async Task<SuseCursor> GetCursorAsync(CancellationToken cancellationToken) + { + var state = await _stateRepository.TryGetAsync(SourceName, cancellationToken).ConfigureAwait(false); + return state is null ? SuseCursor.Empty : SuseCursor.FromBson(state.Cursor); + } + + private async Task UpdateCursorAsync(SuseCursor cursor, CancellationToken cancellationToken) + { + var document = cursor.ToBsonDocument(); + await _stateRepository.UpdateCursorAsync(SourceName, document, _timeProvider.GetUtcNow(), cancellationToken).ConfigureAwait(false); + } + + private static BsonDocument ToBson(SuseAdvisoryDto dto) + { + var packages = new BsonArray(); + foreach (var package in dto.Packages) + { + var packageDoc = new BsonDocument + { + ["package"] = package.Package, + ["platform"] = package.Platform, + ["canonical"] = package.CanonicalNevra, + ["status"] = package.Status + }; + + if (!string.IsNullOrWhiteSpace(package.Architecture)) + { + packageDoc["arch"] = package.Architecture; + } + + if (!string.IsNullOrWhiteSpace(package.IntroducedVersion)) + { + packageDoc["introduced"] = package.IntroducedVersion; + } + + if (!string.IsNullOrWhiteSpace(package.FixedVersion)) + { + packageDoc["fixed"] = package.FixedVersion; + } + + if (!string.IsNullOrWhiteSpace(package.LastAffectedVersion)) + { + packageDoc["last"] = package.LastAffectedVersion; + } + + packages.Add(packageDoc); + } + + var references = new BsonArray(); + foreach (var reference in dto.References) + { + var referenceDoc = new BsonDocument + { + ["url"] = reference.Url + }; + + if (!string.IsNullOrWhiteSpace(reference.Kind)) + { + referenceDoc["kind"] = reference.Kind; + } + + if (!string.IsNullOrWhiteSpace(reference.Title)) + { + referenceDoc["title"] = reference.Title; + } + + references.Add(referenceDoc); + } + + return new BsonDocument + { + ["advisoryId"] = dto.AdvisoryId, + ["title"] = dto.Title ?? string.Empty, + ["summary"] = dto.Summary ?? string.Empty, + ["published"] = dto.Published.UtcDateTime, + ["cves"] = new BsonArray(dto.CveIds ?? Array.Empty<string>()), + ["packages"] = packages, + ["references"] = references + }; + } + + private static SuseAdvisoryDto FromBson(BsonDocument document) + { + var advisoryId = document.GetValue("advisoryId", string.Empty).AsString; + var title = document.GetValue("title", advisoryId).AsString; + var summary = document.TryGetValue("summary", out var summaryValue) ? summaryValue.AsString : null; + var published = document.TryGetValue("published", out var publishedValue) + ? publishedValue.BsonType switch + { + BsonType.DateTime => DateTime.SpecifyKind(publishedValue.ToUniversalTime(), DateTimeKind.Utc), + BsonType.String when DateTimeOffset.TryParse(publishedValue.AsString, out var parsed) => parsed.ToUniversalTime(), + _ => DateTimeOffset.UtcNow + } + : DateTimeOffset.UtcNow; + + var cves = document.TryGetValue("cves", out var cveArray) && cveArray is BsonArray bsonCves + ? bsonCves.OfType<BsonValue>() + .Select(static value => value?.ToString()) + .Where(static value => !string.IsNullOrWhiteSpace(value)) + .Select(static value => value!) + .Distinct(StringComparer.OrdinalIgnoreCase) + .ToArray() + : Array.Empty<string>(); + + var packageList = new List<SusePackageStateDto>(); + if (document.TryGetValue("packages", out var packageArray) && packageArray is BsonArray bsonPackages) + { + foreach (var element in bsonPackages.OfType<BsonDocument>()) + { + var package = element.GetValue("package", string.Empty).AsString; + var platform = element.GetValue("platform", string.Empty).AsString; + var canonical = element.GetValue("canonical", string.Empty).AsString; + var status = element.GetValue("status", "unknown").AsString; + + var architecture = element.TryGetValue("arch", out var archValue) ? archValue.AsString : null; + var introduced = element.TryGetValue("introduced", out var introducedValue) ? introducedValue.AsString : null; + var fixedVersion = element.TryGetValue("fixed", out var fixedValue) ? fixedValue.AsString : null; + var last = element.TryGetValue("last", out var lastValue) ? lastValue.AsString : null; + + packageList.Add(new SusePackageStateDto( + package, + platform, + architecture, + canonical, + introduced, + fixedVersion, + last, + status)); + } + } + + var referenceList = new List<SuseReferenceDto>(); + if (document.TryGetValue("references", out var referenceArray) && referenceArray is BsonArray bsonReferences) + { + foreach (var element in bsonReferences.OfType<BsonDocument>()) + { + var url = element.GetValue("url", string.Empty).AsString; + if (string.IsNullOrWhiteSpace(url)) + { + continue; + } + + referenceList.Add(new SuseReferenceDto( + url, + element.TryGetValue("kind", out var kindValue) ? kindValue.AsString : null, + element.TryGetValue("title", out var titleValue) ? titleValue.AsString : null)); + } + } + + return new SuseAdvisoryDto( + advisoryId, + string.IsNullOrWhiteSpace(title) ? advisoryId : title, + string.IsNullOrWhiteSpace(summary) ? null : summary, + published, + cves, + packageList, + referenceList); + } +} diff --git a/src/StellaOps.Feedser.Source.Distro.Suse/SuseConnectorPlugin.cs b/src/StellaOps.Feedser.Source.Distro.Suse/SuseConnectorPlugin.cs new file mode 100644 index 00000000..00be404e --- /dev/null +++ b/src/StellaOps.Feedser.Source.Distro.Suse/SuseConnectorPlugin.cs @@ -0,0 +1,20 @@ +using System; +using Microsoft.Extensions.DependencyInjection; +using StellaOps.Plugin; + +namespace StellaOps.Feedser.Source.Distro.Suse; + +public sealed class SuseConnectorPlugin : IConnectorPlugin +{ + public const string SourceName = "distro-suse"; + + public string Name => SourceName; + + public bool IsAvailable(IServiceProvider services) => services is not null; + + public IFeedConnector Create(IServiceProvider services) + { + ArgumentNullException.ThrowIfNull(services); + return ActivatorUtilities.CreateInstance<SuseConnector>(services); + } +} diff --git a/src/StellaOps.Feedser.Source.Distro.Suse/SuseDependencyInjectionRoutine.cs b/src/StellaOps.Feedser.Source.Distro.Suse/SuseDependencyInjectionRoutine.cs new file mode 100644 index 00000000..60da18e7 --- /dev/null +++ b/src/StellaOps.Feedser.Source.Distro.Suse/SuseDependencyInjectionRoutine.cs @@ -0,0 +1,53 @@ +using System; +using Microsoft.Extensions.Configuration; +using Microsoft.Extensions.DependencyInjection; +using StellaOps.DependencyInjection; +using StellaOps.Feedser.Core.Jobs; +using StellaOps.Feedser.Source.Distro.Suse.Configuration; + +namespace StellaOps.Feedser.Source.Distro.Suse; + +public sealed class SuseDependencyInjectionRoutine : IDependencyInjectionRoutine +{ + private const string ConfigurationSection = "feedser:sources:suse"; + private const string FetchCron = "*/30 * * * *"; + private const string ParseCron = "5,35 * * * *"; + private const string MapCron = "10,40 * * * *"; + + private static readonly TimeSpan FetchTimeout = TimeSpan.FromMinutes(6); + private static readonly TimeSpan ParseTimeout = TimeSpan.FromMinutes(10); + private static readonly TimeSpan MapTimeout = TimeSpan.FromMinutes(10); + private static readonly TimeSpan LeaseDuration = TimeSpan.FromMinutes(5); + + public IServiceCollection Register(IServiceCollection services, IConfiguration configuration) + { + ArgumentNullException.ThrowIfNull(services); + ArgumentNullException.ThrowIfNull(configuration); + + services.AddSuseConnector(options => + { + configuration.GetSection(ConfigurationSection).Bind(options); + options.Validate(); + }); + + var scheduler = new JobSchedulerBuilder(services); + scheduler + .AddJob<SuseFetchJob>( + SuseJobKinds.Fetch, + cronExpression: FetchCron, + timeout: FetchTimeout, + leaseDuration: LeaseDuration) + .AddJob<SuseParseJob>( + SuseJobKinds.Parse, + cronExpression: ParseCron, + timeout: ParseTimeout, + leaseDuration: LeaseDuration) + .AddJob<SuseMapJob>( + SuseJobKinds.Map, + cronExpression: MapCron, + timeout: MapTimeout, + leaseDuration: LeaseDuration); + + return services; + } +} diff --git a/src/StellaOps.Feedser.Source.Distro.Suse/SuseServiceCollectionExtensions.cs b/src/StellaOps.Feedser.Source.Distro.Suse/SuseServiceCollectionExtensions.cs new file mode 100644 index 00000000..51aff135 --- /dev/null +++ b/src/StellaOps.Feedser.Source.Distro.Suse/SuseServiceCollectionExtensions.cs @@ -0,0 +1,35 @@ +using System; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Options; +using StellaOps.Feedser.Source.Common.Http; +using StellaOps.Feedser.Source.Distro.Suse.Configuration; + +namespace StellaOps.Feedser.Source.Distro.Suse; + +public static class SuseServiceCollectionExtensions +{ + public static IServiceCollection AddSuseConnector(this IServiceCollection services, Action<SuseOptions> configure) + { + ArgumentNullException.ThrowIfNull(services); + ArgumentNullException.ThrowIfNull(configure); + + services.AddOptions<SuseOptions>() + .Configure(configure) + .PostConfigure(static opts => opts.Validate()); + + services.AddSourceHttpClient(SuseOptions.HttpClientName, (sp, httpOptions) => + { + var options = sp.GetRequiredService<IOptions<SuseOptions>>().Value; + httpOptions.BaseAddress = new Uri(options.AdvisoryBaseUri.GetLeftPart(UriPartial.Authority), UriKind.Absolute); + httpOptions.Timeout = options.FetchTimeout; + httpOptions.UserAgent = options.UserAgent; + httpOptions.AllowedHosts.Clear(); + httpOptions.AllowedHosts.Add(options.AdvisoryBaseUri.Host); + httpOptions.AllowedHosts.Add(options.ChangesEndpoint.Host); + httpOptions.DefaultRequestHeaders["Accept"] = "text/csv,application/json;q=0.9,text/plain;q=0.8"; + }); + + services.AddTransient<SuseConnector>(); + return services; + } +} diff --git a/src/StellaOps.Feedser.Source.Distro.Ubuntu.Tests/Fixtures/ubuntu-notices-page0.json b/src/StellaOps.Feedser.Source.Distro.Ubuntu.Tests/Fixtures/ubuntu-notices-page0.json new file mode 100644 index 00000000..f0b95809 --- /dev/null +++ b/src/StellaOps.Feedser.Source.Distro.Ubuntu.Tests/Fixtures/ubuntu-notices-page0.json @@ -0,0 +1,40 @@ +{ + "offset": 0, + "limit": 1, + "total_results": 2, + "notices": [ + { + "id": "USN-9001-1", + "title": "Kernel update", + "summary": "Kernel fixes", + "published": "2025-01-20T08:30:00Z", + "cves_ids": [ + "CVE-2025-2000" + ], + "cves": [ + { + "id": "CVE-2025-2000" + } + ], + "references": [], + "release_packages": { + "noble": [ + { + "name": "linux-image", + "version": "6.8.0-1010.11", + "pocket": "security", + "is_source": false + } + ], + "focal": [ + { + "name": "linux-image", + "version": "5.15.0-200.0", + "pocket": "esm-infra", + "is_source": false + } + ] + } + } + ] +} diff --git a/src/StellaOps.Feedser.Source.Distro.Ubuntu.Tests/Fixtures/ubuntu-notices-page1.json b/src/StellaOps.Feedser.Source.Distro.Ubuntu.Tests/Fixtures/ubuntu-notices-page1.json new file mode 100644 index 00000000..a6cc2cd2 --- /dev/null +++ b/src/StellaOps.Feedser.Source.Distro.Ubuntu.Tests/Fixtures/ubuntu-notices-page1.json @@ -0,0 +1,42 @@ +{ + "offset": 1, + "limit": 1, + "total_results": 2, + "notices": [ + { + "id": "USN-9000-1", + "title": "Example security update", + "summary": "Package fixes", + "published": "2025-01-15T12:00:00Z", + "cves_ids": [ + "CVE-2025-1000", + "CVE-2025-1001" + ], + "cves": [ + { + "id": "CVE-2025-1000" + }, + { + "id": "CVE-2025-1001" + } + ], + "references": [ + { + "url": "https://ubuntu.com/security/USN-9000-1", + "category": "self", + "summary": "USN" + } + ], + "release_packages": { + "jammy": [ + { + "name": "examplepkg", + "version": "1.2.3-0ubuntu0.22.04.1", + "pocket": "security", + "is_source": false + } + ] + } + } + ] +} diff --git a/src/StellaOps.Feedser.Source.Distro.Ubuntu.Tests/StellaOps.Feedser.Source.Distro.Ubuntu.Tests.csproj b/src/StellaOps.Feedser.Source.Distro.Ubuntu.Tests/StellaOps.Feedser.Source.Distro.Ubuntu.Tests.csproj new file mode 100644 index 00000000..65ac759c --- /dev/null +++ b/src/StellaOps.Feedser.Source.Distro.Ubuntu.Tests/StellaOps.Feedser.Source.Distro.Ubuntu.Tests.csproj @@ -0,0 +1,18 @@ +<Project Sdk="Microsoft.NET.Sdk"> + <PropertyGroup> + <TargetFramework>net10.0</TargetFramework> + <ImplicitUsings>enable</ImplicitUsings> + <Nullable>enable</Nullable> + </PropertyGroup> + <ItemGroup> + <ProjectReference Include="../StellaOps.Feedser.Source.Distro.Ubuntu/StellaOps.Feedser.Source.Distro.Ubuntu.csproj" /> + <ProjectReference Include="../StellaOps.Feedser.Source.Common/StellaOps.Feedser.Source.Common.csproj" /> + <ProjectReference Include="../StellaOps.Feedser.Models/StellaOps.Feedser.Models.csproj" /> + <ProjectReference Include="../StellaOps.Feedser.Storage.Mongo/StellaOps.Feedser.Storage.Mongo.csproj" /> + </ItemGroup> + <ItemGroup> + <None Update="Fixtures\**\*"> + <CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory> + </None> + </ItemGroup> +</Project> diff --git a/src/StellaOps.Feedser.Source.Distro.Ubuntu.Tests/UbuntuConnectorTests.cs b/src/StellaOps.Feedser.Source.Distro.Ubuntu.Tests/UbuntuConnectorTests.cs new file mode 100644 index 00000000..b84c7f35 --- /dev/null +++ b/src/StellaOps.Feedser.Source.Distro.Ubuntu.Tests/UbuntuConnectorTests.cs @@ -0,0 +1,171 @@ +using System; +using System.IO; +using System.Linq; +using System.Net; +using System.Net.Http; +using System.Net.Http.Headers; +using System.Text; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Http; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Logging.Abstractions; +using Microsoft.Extensions.Time.Testing; +using StellaOps.Feedser.Models; +using StellaOps.Feedser.Source.Common; +using StellaOps.Feedser.Source.Common.Http; +using StellaOps.Feedser.Source.Common.Testing; +using StellaOps.Feedser.Source.Distro.Ubuntu; +using StellaOps.Feedser.Source.Distro.Ubuntu.Configuration; +using StellaOps.Feedser.Storage.Mongo; +using StellaOps.Feedser.Storage.Mongo.Advisories; +using StellaOps.Feedser.Testing; +using Xunit; + +namespace StellaOps.Feedser.Source.Distro.Ubuntu.Tests; + +[Collection("mongo-fixture")] +public sealed class UbuntuConnectorTests : IAsyncLifetime +{ + private static readonly Uri IndexPage0Uri = new("https://ubuntu.com/security/notices.json?offset=0&limit=1"); + private static readonly Uri IndexPage1Uri = new("https://ubuntu.com/security/notices.json?offset=1&limit=1"); + + private readonly MongoIntegrationFixture _fixture; + private readonly FakeTimeProvider _timeProvider; + private readonly CannedHttpMessageHandler _handler; + + public UbuntuConnectorTests(MongoIntegrationFixture fixture) + { + _fixture = fixture; + _timeProvider = new FakeTimeProvider(new DateTimeOffset(2025, 1, 25, 0, 0, 0, TimeSpan.Zero)); + _handler = new CannedHttpMessageHandler(); + } + + [Fact] + public async Task FetchParseMap_GeneratesEvrRangePrimitives() + { + await using var provider = await BuildServiceProviderAsync(); + + SeedInitialResponses(); + + var connector = provider.GetRequiredService<UbuntuConnector>(); + await connector.FetchAsync(provider, CancellationToken.None); + _timeProvider.Advance(TimeSpan.FromMinutes(1)); + await connector.ParseAsync(provider, CancellationToken.None); + await connector.MapAsync(provider, CancellationToken.None); + + var advisoryStore = provider.GetRequiredService<IAdvisoryStore>(); + var advisories = await advisoryStore.GetRecentAsync(10, CancellationToken.None); + Assert.Equal(2, advisories.Count); + + var kernelNotice = advisories.Single(a => a.AdvisoryKey == "USN-9001-1"); + var noblePackage = Assert.Single(kernelNotice.AffectedPackages, pkg => pkg.Platform == "noble"); + var range = Assert.Single(noblePackage.VersionRanges); + Assert.Equal("evr", range.RangeKind); + Assert.NotNull(range.Primitives); + Assert.NotNull(range.Primitives!.Evr?.Fixed); + Assert.Contains("CVE-2025-2000", kernelNotice.Aliases); + + SeedNotModifiedResponses(); + + await connector.FetchAsync(provider, CancellationToken.None); + _timeProvider.Advance(TimeSpan.FromMinutes(1)); + await connector.ParseAsync(provider, CancellationToken.None); + await connector.MapAsync(provider, CancellationToken.None); + + advisories = await advisoryStore.GetRecentAsync(10, CancellationToken.None); + Assert.Equal(2, advisories.Count); + _handler.AssertNoPendingResponses(); + } + + private async Task<ServiceProvider> BuildServiceProviderAsync() + { + await _fixture.Client.DropDatabaseAsync(_fixture.Database.DatabaseNamespace.DatabaseName); + _handler.Clear(); + + var services = new ServiceCollection(); + services.AddLogging(builder => builder.AddProvider(NullLoggerProvider.Instance)); + services.AddSingleton<TimeProvider>(_timeProvider); + services.AddSingleton(_handler); + + services.AddMongoStorage(options => + { + options.ConnectionString = _fixture.Runner.ConnectionString; + options.DatabaseName = _fixture.Database.DatabaseNamespace.DatabaseName; + options.CommandTimeout = TimeSpan.FromSeconds(5); + }); + + services.AddSourceCommon(); + services.AddUbuntuConnector(options => + { + options.NoticesEndpoint = new Uri("https://ubuntu.com/security/notices.json"); + options.NoticeDetailBaseUri = new Uri("https://ubuntu.com/security/"); + options.MaxNoticesPerFetch = 2; + options.IndexPageSize = 1; + }); + + services.Configure<HttpClientFactoryOptions>(UbuntuOptions.HttpClientName, builderOptions => + { + builderOptions.HttpMessageHandlerBuilderActions.Add(builder => + { + builder.PrimaryHandler = _handler; + }); + }); + + var provider = services.BuildServiceProvider(); + var bootstrapper = provider.GetRequiredService<MongoBootstrapper>(); + await bootstrapper.InitializeAsync(CancellationToken.None); + return provider; + } + + private void SeedInitialResponses() + { + _handler.AddResponse(IndexPage0Uri, () => + { + var response = new HttpResponseMessage(HttpStatusCode.OK) + { + Content = new StringContent(ReadFixture("Fixtures/ubuntu-notices-page0.json"), Encoding.UTF8, "application/json") + }; + response.Headers.ETag = new EntityTagHeaderValue("\"index-page0-v1\""); + return response; + }); + + _handler.AddResponse(IndexPage1Uri, () => + { + var response = new HttpResponseMessage(HttpStatusCode.OK) + { + Content = new StringContent(ReadFixture("Fixtures/ubuntu-notices-page1.json"), Encoding.UTF8, "application/json") + }; + response.Headers.ETag = new EntityTagHeaderValue("\"index-page1-v1\""); + return response; + }); + } + + private void SeedNotModifiedResponses() + { + _handler.AddResponse(IndexPage0Uri, () => + { + var response = new HttpResponseMessage(HttpStatusCode.NotModified); + response.Headers.ETag = new EntityTagHeaderValue("\"index-page0-v1\""); + return response; + }); + + // Page 1 remains cached; the connector should skip fetching it when page 0 is unchanged. + } + + private static string ReadFixture(string relativePath) + { + var path = Path.Combine(AppContext.BaseDirectory, relativePath.Replace('/', Path.DirectorySeparatorChar)); + if (!File.Exists(path)) + { + throw new FileNotFoundException($"Fixture '{relativePath}' not found.", path); + } + + return File.ReadAllText(path); + } + + public Task InitializeAsync() => Task.CompletedTask; + + public Task DisposeAsync() => Task.CompletedTask; +} diff --git a/src/StellaOps.Feedser.Source.Distro.Ubuntu/Configuration/UbuntuOptions.cs b/src/StellaOps.Feedser.Source.Distro.Ubuntu/Configuration/UbuntuOptions.cs new file mode 100644 index 00000000..97596c2f --- /dev/null +++ b/src/StellaOps.Feedser.Source.Distro.Ubuntu/Configuration/UbuntuOptions.cs @@ -0,0 +1,69 @@ +using System; + +namespace StellaOps.Feedser.Source.Distro.Ubuntu.Configuration; + +public sealed class UbuntuOptions +{ + public const string HttpClientName = "feedser.ubuntu"; + public const int MaxPageSize = 20; + + /// <summary> + /// Endpoint exposing the rolling JSON index of Ubuntu Security Notices. + /// </summary> + public Uri NoticesEndpoint { get; set; } = new("https://ubuntu.com/security/notices.json"); + + /// <summary> + /// Base URI where individual notice detail pages live. + /// </summary> + public Uri NoticeDetailBaseUri { get; set; } = new("https://ubuntu.com/security/"); + + public TimeSpan FetchTimeout { get; set; } = TimeSpan.FromSeconds(45); + + public TimeSpan InitialBackfill { get; set; } = TimeSpan.FromDays(30); + + public TimeSpan ResumeOverlap { get; set; } = TimeSpan.FromDays(3); + + public int MaxNoticesPerFetch { get; set; } = 60; + + public int IndexPageSize { get; set; } = 20; + + public string UserAgent { get; set; } = "StellaOps.Feedser.Ubuntu/0.1 (+https://stella-ops.org)"; + + public void Validate() + { + if (NoticesEndpoint is null || !NoticesEndpoint.IsAbsoluteUri) + { + throw new InvalidOperationException("Ubuntu notices endpoint must be an absolute URI."); + } + + if (NoticeDetailBaseUri is null || !NoticeDetailBaseUri.IsAbsoluteUri) + { + throw new InvalidOperationException("Ubuntu notice detail base URI must be an absolute URI."); + } + + if (MaxNoticesPerFetch <= 0 || MaxNoticesPerFetch > 200) + { + throw new InvalidOperationException("MaxNoticesPerFetch must be between 1 and 200."); + } + + if (FetchTimeout <= TimeSpan.Zero || FetchTimeout > TimeSpan.FromMinutes(5)) + { + throw new InvalidOperationException("FetchTimeout must be positive and less than five minutes."); + } + + if (InitialBackfill < TimeSpan.Zero || InitialBackfill > TimeSpan.FromDays(365)) + { + throw new InvalidOperationException("InitialBackfill must be between 0 and 365 days."); + } + + if (ResumeOverlap < TimeSpan.Zero || ResumeOverlap > TimeSpan.FromDays(14)) + { + throw new InvalidOperationException("ResumeOverlap must be between 0 and 14 days."); + } + + if (IndexPageSize <= 0 || IndexPageSize > MaxPageSize) + { + throw new InvalidOperationException($"IndexPageSize must be between 1 and {MaxPageSize}."); + } + } +} diff --git a/src/StellaOps.Feedser.Source.Distro.Ubuntu/Internal/UbuntuCursor.cs b/src/StellaOps.Feedser.Source.Distro.Ubuntu/Internal/UbuntuCursor.cs new file mode 100644 index 00000000..56d3bb63 --- /dev/null +++ b/src/StellaOps.Feedser.Source.Distro.Ubuntu/Internal/UbuntuCursor.cs @@ -0,0 +1,177 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using MongoDB.Bson; + +namespace StellaOps.Feedser.Source.Distro.Ubuntu.Internal; + +internal sealed record UbuntuCursor( + DateTimeOffset? LastPublished, + IReadOnlyCollection<string> ProcessedNoticeIds, + IReadOnlyCollection<Guid> PendingDocuments, + IReadOnlyCollection<Guid> PendingMappings, + IReadOnlyDictionary<string, UbuntuFetchCacheEntry> FetchCache) +{ + private static readonly IReadOnlyCollection<string> EmptyIds = Array.Empty<string>(); + private static readonly IReadOnlyCollection<Guid> EmptyGuidList = Array.Empty<Guid>(); + private static readonly IReadOnlyDictionary<string, UbuntuFetchCacheEntry> EmptyCache = + new Dictionary<string, UbuntuFetchCacheEntry>(StringComparer.OrdinalIgnoreCase); + + public static UbuntuCursor Empty { get; } = new(null, EmptyIds, EmptyGuidList, EmptyGuidList, EmptyCache); + + public static UbuntuCursor FromBson(BsonDocument? document) + { + if (document is null || document.ElementCount == 0) + { + return Empty; + } + + DateTimeOffset? lastPublished = null; + if (document.TryGetValue("lastPublished", out var value)) + { + lastPublished = value.BsonType switch + { + BsonType.DateTime => DateTime.SpecifyKind(value.ToUniversalTime(), DateTimeKind.Utc), + BsonType.String when DateTimeOffset.TryParse(value.AsString, out var parsed) => parsed.ToUniversalTime(), + _ => null + }; + } + + var processed = ReadStringSet(document, "processedIds"); + var pendingDocuments = ReadGuidSet(document, "pendingDocuments"); + var pendingMappings = ReadGuidSet(document, "pendingMappings"); + var cache = ReadCache(document); + + return new UbuntuCursor(lastPublished, processed, pendingDocuments, pendingMappings, cache); + } + + public BsonDocument ToBsonDocument() + { + var doc = new BsonDocument + { + ["pendingDocuments"] = new BsonArray(PendingDocuments.Select(id => id.ToString())), + ["pendingMappings"] = new BsonArray(PendingMappings.Select(id => id.ToString())) + }; + + if (LastPublished.HasValue) + { + doc["lastPublished"] = LastPublished.Value.UtcDateTime; + } + + if (ProcessedNoticeIds.Count > 0) + { + doc["processedIds"] = new BsonArray(ProcessedNoticeIds); + } + + if (FetchCache.Count > 0) + { + var cacheDoc = new BsonDocument(); + foreach (var (key, entry) in FetchCache) + { + cacheDoc[key] = entry.ToBsonDocument(); + } + + doc["fetchCache"] = cacheDoc; + } + + return doc; + } + + public UbuntuCursor WithPendingDocuments(IEnumerable<Guid> ids) + => this with { PendingDocuments = ids?.Distinct().ToArray() ?? EmptyGuidList }; + + public UbuntuCursor WithPendingMappings(IEnumerable<Guid> ids) + => this with { PendingMappings = ids?.Distinct().ToArray() ?? EmptyGuidList }; + + public UbuntuCursor WithFetchCache(IDictionary<string, UbuntuFetchCacheEntry>? cache) + { + if (cache is null || cache.Count == 0) + { + return this with { FetchCache = EmptyCache }; + } + + return this with { FetchCache = new Dictionary<string, UbuntuFetchCacheEntry>(cache, StringComparer.OrdinalIgnoreCase) }; + } + + public UbuntuCursor WithProcessed(DateTimeOffset published, IEnumerable<string> ids) + => this with + { + LastPublished = published.ToUniversalTime(), + ProcessedNoticeIds = ids?.Where(static id => !string.IsNullOrWhiteSpace(id)) + .Select(static id => id.Trim()) + .Distinct(StringComparer.OrdinalIgnoreCase) + .ToArray() ?? EmptyIds + }; + + public bool TryGetCache(string key, out UbuntuFetchCacheEntry entry) + { + if (FetchCache.Count == 0) + { + entry = UbuntuFetchCacheEntry.Empty; + return false; + } + + return FetchCache.TryGetValue(key, out entry!); + } + + private static IReadOnlyCollection<string> ReadStringSet(BsonDocument document, string field) + { + if (!document.TryGetValue(field, out var value) || value is not BsonArray array) + { + return EmptyIds; + } + + var list = new List<string>(array.Count); + foreach (var element in array) + { + if (element.BsonType == BsonType.String) + { + var str = element.AsString.Trim(); + if (!string.IsNullOrWhiteSpace(str)) + { + list.Add(str); + } + } + } + + return list; + } + + private static IReadOnlyCollection<Guid> ReadGuidSet(BsonDocument document, string field) + { + if (!document.TryGetValue(field, out var value) || value is not BsonArray array) + { + return EmptyGuidList; + } + + var list = new List<Guid>(array.Count); + foreach (var element in array) + { + if (Guid.TryParse(element.ToString(), out var guid)) + { + list.Add(guid); + } + } + + return list; + } + + private static IReadOnlyDictionary<string, UbuntuFetchCacheEntry> ReadCache(BsonDocument document) + { + if (!document.TryGetValue("fetchCache", out var value) || value is not BsonDocument cacheDoc || cacheDoc.ElementCount == 0) + { + return EmptyCache; + } + + var cache = new Dictionary<string, UbuntuFetchCacheEntry>(StringComparer.OrdinalIgnoreCase); + foreach (var element in cacheDoc.Elements) + { + if (element.Value is BsonDocument entryDoc) + { + cache[element.Name] = UbuntuFetchCacheEntry.FromBson(entryDoc); + } + } + + return cache; + } +} diff --git a/src/StellaOps.Feedser.Source.Distro.Ubuntu/Internal/UbuntuFetchCacheEntry.cs b/src/StellaOps.Feedser.Source.Distro.Ubuntu/Internal/UbuntuFetchCacheEntry.cs new file mode 100644 index 00000000..29d739af --- /dev/null +++ b/src/StellaOps.Feedser.Source.Distro.Ubuntu/Internal/UbuntuFetchCacheEntry.cs @@ -0,0 +1,76 @@ +using System; +using MongoDB.Bson; + +namespace StellaOps.Feedser.Source.Distro.Ubuntu.Internal; + +internal sealed record UbuntuFetchCacheEntry(string? ETag, DateTimeOffset? LastModified) +{ + public static UbuntuFetchCacheEntry Empty { get; } = new(null, null); + + public static UbuntuFetchCacheEntry FromDocument(StellaOps.Feedser.Storage.Mongo.Documents.DocumentRecord document) + => new(document.Etag, document.LastModified); + + public static UbuntuFetchCacheEntry FromBson(BsonDocument document) + { + if (document is null || document.ElementCount == 0) + { + return Empty; + } + + string? etag = null; + DateTimeOffset? lastModified = null; + + if (document.TryGetValue("etag", out var etagValue) && etagValue.BsonType == BsonType.String) + { + etag = etagValue.AsString; + } + + if (document.TryGetValue("lastModified", out var modifiedValue)) + { + lastModified = modifiedValue.BsonType switch + { + BsonType.DateTime => DateTime.SpecifyKind(modifiedValue.ToUniversalTime(), DateTimeKind.Utc), + BsonType.String when DateTimeOffset.TryParse(modifiedValue.AsString, out var parsed) => parsed.ToUniversalTime(), + _ => null + }; + } + + return new UbuntuFetchCacheEntry(etag, lastModified); + } + + public BsonDocument ToBsonDocument() + { + var doc = new BsonDocument(); + if (!string.IsNullOrWhiteSpace(ETag)) + { + doc["etag"] = ETag; + } + + if (LastModified.HasValue) + { + doc["lastModified"] = LastModified.Value.UtcDateTime; + } + + return doc; + } + + public bool Matches(StellaOps.Feedser.Storage.Mongo.Documents.DocumentRecord document) + { + if (document is null) + { + return false; + } + + if (!string.Equals(ETag, document.Etag, StringComparison.Ordinal)) + { + return false; + } + + if (LastModified.HasValue && document.LastModified.HasValue) + { + return LastModified.Value.UtcDateTime == document.LastModified.Value.UtcDateTime; + } + + return !LastModified.HasValue && !document.LastModified.HasValue; + } +} diff --git a/src/StellaOps.Feedser.Source.Distro.Ubuntu/Internal/UbuntuMapper.cs b/src/StellaOps.Feedser.Source.Distro.Ubuntu/Internal/UbuntuMapper.cs new file mode 100644 index 00000000..1137fc37 --- /dev/null +++ b/src/StellaOps.Feedser.Source.Distro.Ubuntu/Internal/UbuntuMapper.cs @@ -0,0 +1,217 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using StellaOps.Feedser.Models; +using StellaOps.Feedser.Normalization.Distro; +using StellaOps.Feedser.Storage.Mongo.Documents; + +namespace StellaOps.Feedser.Source.Distro.Ubuntu.Internal; + +internal static class UbuntuMapper +{ + public static Advisory Map(UbuntuNoticeDto dto, DocumentRecord document, DateTimeOffset recordedAt) + { + ArgumentNullException.ThrowIfNull(dto); + ArgumentNullException.ThrowIfNull(document); + + var aliases = BuildAliases(dto); + var references = BuildReferences(dto, recordedAt); + var packages = BuildPackages(dto, recordedAt); + + var fetchProvenance = new AdvisoryProvenance( + UbuntuConnectorPlugin.SourceName, + "document", + document.Uri, + document.FetchedAt.ToUniversalTime()); + + var mapProvenance = new AdvisoryProvenance( + UbuntuConnectorPlugin.SourceName, + "mapping", + dto.NoticeId, + recordedAt); + + return new Advisory( + advisoryKey: dto.NoticeId, + title: dto.Title ?? dto.NoticeId, + summary: dto.Summary, + language: "en", + published: dto.Published, + modified: recordedAt > dto.Published ? recordedAt : dto.Published, + severity: null, + exploitKnown: false, + aliases: aliases, + references: references, + affectedPackages: packages, + cvssMetrics: Array.Empty<CvssMetric>(), + provenance: new[] { fetchProvenance, mapProvenance }); + } + + private static string[] BuildAliases(UbuntuNoticeDto dto) + { + var aliases = new HashSet<string>(StringComparer.OrdinalIgnoreCase) + { + dto.NoticeId + }; + + foreach (var cve in dto.CveIds ?? Array.Empty<string>()) + { + if (!string.IsNullOrWhiteSpace(cve)) + { + aliases.Add(cve.Trim()); + } + } + + return aliases.OrderBy(static alias => alias, StringComparer.OrdinalIgnoreCase).ToArray(); + } + + private static AdvisoryReference[] BuildReferences(UbuntuNoticeDto dto, DateTimeOffset recordedAt) + { + if (dto.References is null || dto.References.Count == 0) + { + return Array.Empty<AdvisoryReference>(); + } + + var references = new List<AdvisoryReference>(dto.References.Count); + foreach (var reference in dto.References) + { + if (string.IsNullOrWhiteSpace(reference.Url)) + { + continue; + } + + try + { + var provenance = new AdvisoryProvenance( + UbuntuConnectorPlugin.SourceName, + "reference", + reference.Url, + recordedAt); + + references.Add(new AdvisoryReference( + reference.Url.Trim(), + NormalizeReferenceKind(reference.Kind), + reference.Kind, + reference.Title, + provenance)); + } + catch (ArgumentException) + { + // ignore poorly formed URIs + } + } + + return references.Count == 0 + ? Array.Empty<AdvisoryReference>() + : references + .OrderBy(static reference => reference.Url, StringComparer.OrdinalIgnoreCase) + .ToArray(); + } + + private static string? NormalizeReferenceKind(string? kind) + { + if (string.IsNullOrWhiteSpace(kind)) + { + return null; + } + + return kind.Trim().ToLowerInvariant() switch + { + "external" => "external", + "self" => "advisory", + _ => null + }; + } + + private static IReadOnlyList<AffectedPackage> BuildPackages(UbuntuNoticeDto dto, DateTimeOffset recordedAt) + { + if (dto.Packages is null || dto.Packages.Count == 0) + { + return Array.Empty<AffectedPackage>(); + } + + var list = new List<AffectedPackage>(); + foreach (var package in dto.Packages) + { + if (string.IsNullOrWhiteSpace(package.Package) || string.IsNullOrWhiteSpace(package.Version)) + { + continue; + } + + if (!DebianEvr.TryParse(package.Version, out var evr) || evr is null) + { + continue; + } + + var provenance = new AdvisoryProvenance( + UbuntuConnectorPlugin.SourceName, + "affected", + $"{dto.NoticeId}:{package.Release}:{package.Package}", + recordedAt); + + var rangeProvenance = new AdvisoryProvenance( + UbuntuConnectorPlugin.SourceName, + "range", + $"{dto.NoticeId}:{package.Release}:{package.Package}", + recordedAt); + + var rangeExpression = $"fixed:{package.Version}"; + + var extensions = new Dictionary<string, string>(StringComparer.Ordinal) + { + ["ubuntu.release"] = package.Release, + ["ubuntu.pocket"] = package.Pocket ?? string.Empty + }; + + var range = new AffectedVersionRange( + rangeKind: "evr", + introducedVersion: null, + fixedVersion: package.Version, + lastAffectedVersion: null, + rangeExpression: rangeExpression, + provenance: rangeProvenance, + primitives: new RangePrimitives( + SemVer: null, + Nevra: null, + Evr: new EvrPrimitive( + Introduced: null, + Fixed: new EvrComponent(evr.Epoch, evr.Version, evr.Revision.Length == 0 ? null : evr.Revision), + LastAffected: null), + VendorExtensions: extensions)); + + var statuses = new[] + { + new AffectedPackageStatus(DetermineStatus(package), provenance) + }; + + list.Add(new AffectedPackage( + type: AffectedPackageTypes.Deb, + identifier: package.Package, + platform: package.Release, + versionRanges: new[] { range }, + statuses: statuses, + provenance: new[] { provenance })); + } + + return list.Count == 0 + ? Array.Empty<AffectedPackage>() + : list + .OrderBy(static pkg => pkg.Platform, StringComparer.OrdinalIgnoreCase) + .ThenBy(static pkg => pkg.Identifier, StringComparer.OrdinalIgnoreCase) + .ToArray(); + } + + private static string DetermineStatus(UbuntuReleasePackageDto package) + { + if (!string.IsNullOrWhiteSpace(package.Pocket) && package.Pocket.Contains("security", StringComparison.OrdinalIgnoreCase)) + { + return "resolved"; + } + + if (!string.IsNullOrWhiteSpace(package.Pocket) && package.Pocket.Contains("esm", StringComparison.OrdinalIgnoreCase)) + { + return "resolved"; + } + + return "resolved"; + } +} diff --git a/src/StellaOps.Feedser.Source.Distro.Ubuntu/Internal/UbuntuNoticeDto.cs b/src/StellaOps.Feedser.Source.Distro.Ubuntu/Internal/UbuntuNoticeDto.cs new file mode 100644 index 00000000..18b8afa7 --- /dev/null +++ b/src/StellaOps.Feedser.Source.Distro.Ubuntu/Internal/UbuntuNoticeDto.cs @@ -0,0 +1,25 @@ +using System; +using System.Collections.Generic; + +namespace StellaOps.Feedser.Source.Distro.Ubuntu.Internal; + +internal sealed record UbuntuNoticeDto( + string NoticeId, + DateTimeOffset Published, + string Title, + string Summary, + IReadOnlyList<string> CveIds, + IReadOnlyList<UbuntuReleasePackageDto> Packages, + IReadOnlyList<UbuntuReferenceDto> References); + +internal sealed record UbuntuReleasePackageDto( + string Release, + string Package, + string Version, + string Pocket, + bool IsSource); + +internal sealed record UbuntuReferenceDto( + string Url, + string? Kind, + string? Title); diff --git a/src/StellaOps.Feedser.Source.Distro.Ubuntu/Internal/UbuntuNoticeParser.cs b/src/StellaOps.Feedser.Source.Distro.Ubuntu/Internal/UbuntuNoticeParser.cs new file mode 100644 index 00000000..3a175e44 --- /dev/null +++ b/src/StellaOps.Feedser.Source.Distro.Ubuntu/Internal/UbuntuNoticeParser.cs @@ -0,0 +1,215 @@ +using System; +using System.Collections.Generic; +using System.Globalization; +using System.Text.Json; + +namespace StellaOps.Feedser.Source.Distro.Ubuntu.Internal; + +internal static class UbuntuNoticeParser +{ + public static UbuntuIndexResponse ParseIndex(string json) + { + ArgumentException.ThrowIfNullOrEmpty(json); + + using var document = JsonDocument.Parse(json); + var root = document.RootElement; + if (!root.TryGetProperty("notices", out var noticesElement) || noticesElement.ValueKind != JsonValueKind.Array) + { + return UbuntuIndexResponse.Empty; + } + + var notices = new List<UbuntuNoticeDto>(noticesElement.GetArrayLength()); + foreach (var noticeElement in noticesElement.EnumerateArray()) + { + if (!noticeElement.TryGetProperty("id", out var idElement)) + { + continue; + } + + var noticeId = idElement.GetString(); + if (string.IsNullOrWhiteSpace(noticeId)) + { + continue; + } + + var published = ParseDate(noticeElement, "published") ?? DateTimeOffset.UtcNow; + var title = noticeElement.TryGetProperty("title", out var titleElement) + ? titleElement.GetString() ?? noticeId + : noticeId; + + var summary = noticeElement.TryGetProperty("summary", out var summaryElement) + ? summaryElement.GetString() ?? string.Empty + : string.Empty; + + var cves = ExtractCves(noticeElement); + var references = ExtractReferences(noticeElement); + var packages = ExtractPackages(noticeElement); + + if (packages.Count == 0) + { + continue; + } + + notices.Add(new UbuntuNoticeDto( + noticeId, + published, + title, + summary, + cves, + packages, + references)); + } + + var offset = root.TryGetProperty("offset", out var offsetElement) && offsetElement.ValueKind == JsonValueKind.Number + ? offsetElement.GetInt32() + : 0; + + var limit = root.TryGetProperty("limit", out var limitElement) && limitElement.ValueKind == JsonValueKind.Number + ? limitElement.GetInt32() + : noticesElement.GetArrayLength(); + + var totalResults = root.TryGetProperty("total_results", out var totalElement) && totalElement.ValueKind == JsonValueKind.Number + ? totalElement.GetInt32() + : notices.Count; + + return new UbuntuIndexResponse(offset, limit, totalResults, notices); + } + + private static IReadOnlyList<string> ExtractCves(JsonElement noticeElement) + { + if (!noticeElement.TryGetProperty("cves", out var cveArray) || cveArray.ValueKind != JsonValueKind.Array) + { + return Array.Empty<string>(); + } + + var set = new HashSet<string>(StringComparer.OrdinalIgnoreCase); + foreach (var cveElement in cveArray.EnumerateArray()) + { + var cve = cveElement.TryGetProperty("id", out var idElement) + ? idElement.GetString() + : cveElement.GetString(); + + if (!string.IsNullOrWhiteSpace(cve)) + { + set.Add(cve.Trim()); + } + } + + if (set.Count == 0) + { + return Array.Empty<string>(); + } + + var list = new List<string>(set); + list.Sort(StringComparer.OrdinalIgnoreCase); + return list; + } + + private static IReadOnlyList<UbuntuReferenceDto> ExtractReferences(JsonElement noticeElement) + { + if (!noticeElement.TryGetProperty("references", out var referencesElement) || referencesElement.ValueKind != JsonValueKind.Array) + { + return Array.Empty<UbuntuReferenceDto>(); + } + + var list = new List<UbuntuReferenceDto>(); + var seen = new HashSet<string>(StringComparer.OrdinalIgnoreCase); + foreach (var referenceElement in referencesElement.EnumerateArray()) + { + var url = referenceElement.TryGetProperty("url", out var urlElement) + ? urlElement.GetString() + : null; + + if (string.IsNullOrWhiteSpace(url) || !seen.Add(url)) + { + continue; + } + + var kind = referenceElement.TryGetProperty("category", out var categoryElement) + ? categoryElement.GetString() + : null; + + var title = referenceElement.TryGetProperty("summary", out var summaryElement) + ? summaryElement.GetString() + : null; + + list.Add(new UbuntuReferenceDto(url.Trim(), kind, title)); + } + + return list.Count == 0 ? Array.Empty<UbuntuReferenceDto>() : list; + } + + private static IReadOnlyList<UbuntuReleasePackageDto> ExtractPackages(JsonElement noticeElement) + { + if (!noticeElement.TryGetProperty("release_packages", out var releasesElement) || releasesElement.ValueKind != JsonValueKind.Object) + { + return Array.Empty<UbuntuReleasePackageDto>(); + } + + var packages = new List<UbuntuReleasePackageDto>(); + foreach (var releaseProperty in releasesElement.EnumerateObject()) + { + var release = releaseProperty.Name; + var packageArray = releaseProperty.Value; + if (packageArray.ValueKind != JsonValueKind.Array) + { + continue; + } + + foreach (var packageElement in packageArray.EnumerateArray()) + { + var name = packageElement.TryGetProperty("name", out var nameElement) + ? nameElement.GetString() + : null; + + var version = packageElement.TryGetProperty("version", out var versionElement) + ? versionElement.GetString() + : null; + + if (string.IsNullOrWhiteSpace(name) || string.IsNullOrWhiteSpace(version)) + { + continue; + } + + var pocket = packageElement.TryGetProperty("pocket", out var pocketElement) + ? pocketElement.GetString() ?? string.Empty + : string.Empty; + + var isSource = packageElement.TryGetProperty("is_source", out var sourceElement) + && sourceElement.ValueKind == JsonValueKind.True; + + packages.Add(new UbuntuReleasePackageDto( + release, + name.Trim(), + version.Trim(), + pocket.Trim(), + isSource)); + } + } + + return packages.Count == 0 ? Array.Empty<UbuntuReleasePackageDto>() : packages; + } + + private static DateTimeOffset? ParseDate(JsonElement element, string propertyName) + { + if (!element.TryGetProperty(propertyName, out var dateElement) || dateElement.ValueKind != JsonValueKind.String) + { + return null; + } + + var value = dateElement.GetString(); + if (string.IsNullOrWhiteSpace(value)) + { + return null; + } + + return DateTimeOffset.TryParse(value, CultureInfo.InvariantCulture, DateTimeStyles.AssumeUniversal, out var parsed) + ? parsed.ToUniversalTime() + : null; + } +} + +internal sealed record UbuntuIndexResponse(int Offset, int Limit, int TotalResults, IReadOnlyList<UbuntuNoticeDto> Notices) +{ + public static UbuntuIndexResponse Empty { get; } = new(0, 0, 0, Array.Empty<UbuntuNoticeDto>()); +} diff --git a/src/StellaOps.Feedser.Source.Distro.Ubuntu/Jobs.cs b/src/StellaOps.Feedser.Source.Distro.Ubuntu/Jobs.cs new file mode 100644 index 00000000..5fd98027 --- /dev/null +++ b/src/StellaOps.Feedser.Source.Distro.Ubuntu/Jobs.cs @@ -0,0 +1,46 @@ +using System; +using System.Threading; +using System.Threading.Tasks; +using StellaOps.Feedser.Core.Jobs; + +namespace StellaOps.Feedser.Source.Distro.Ubuntu; + +internal static class UbuntuJobKinds +{ + public const string Fetch = "source:ubuntu:fetch"; + public const string Parse = "source:ubuntu:parse"; + public const string Map = "source:ubuntu:map"; +} + +internal sealed class UbuntuFetchJob : IJob +{ + private readonly UbuntuConnector _connector; + + public UbuntuFetchJob(UbuntuConnector connector) + => _connector = connector ?? throw new ArgumentNullException(nameof(connector)); + + public Task ExecuteAsync(JobExecutionContext context, CancellationToken cancellationToken) + => _connector.FetchAsync(context.Services, cancellationToken); +} + +internal sealed class UbuntuParseJob : IJob +{ + private readonly UbuntuConnector _connector; + + public UbuntuParseJob(UbuntuConnector connector) + => _connector = connector ?? throw new ArgumentNullException(nameof(connector)); + + public Task ExecuteAsync(JobExecutionContext context, CancellationToken cancellationToken) + => _connector.ParseAsync(context.Services, cancellationToken); +} + +internal sealed class UbuntuMapJob : IJob +{ + private readonly UbuntuConnector _connector; + + public UbuntuMapJob(UbuntuConnector connector) + => _connector = connector ?? throw new ArgumentNullException(nameof(connector)); + + public Task ExecuteAsync(JobExecutionContext context, CancellationToken cancellationToken) + => _connector.MapAsync(context.Services, cancellationToken); +} diff --git a/src/StellaOps.Feedser.Source.Distro.Ubuntu/StellaOps.Feedser.Source.Distro.Ubuntu.csproj b/src/StellaOps.Feedser.Source.Distro.Ubuntu/StellaOps.Feedser.Source.Distro.Ubuntu.csproj new file mode 100644 index 00000000..96165c66 --- /dev/null +++ b/src/StellaOps.Feedser.Source.Distro.Ubuntu/StellaOps.Feedser.Source.Distro.Ubuntu.csproj @@ -0,0 +1,17 @@ +<Project Sdk="Microsoft.NET.Sdk"> + + <PropertyGroup> + <TargetFramework>net10.0</TargetFramework> + <ImplicitUsings>enable</ImplicitUsings> + <Nullable>enable</Nullable> + </PropertyGroup> + + <ItemGroup> + <ProjectReference Include="../StellaOps.Plugin/StellaOps.Plugin.csproj" /> + + <ProjectReference Include="../StellaOps.Feedser.Source.Common/StellaOps.Feedser.Source.Common.csproj" /> + <ProjectReference Include="../StellaOps.Feedser.Models/StellaOps.Feedser.Models.csproj" /> + <ProjectReference Include="../StellaOps.Feedser.Normalization/StellaOps.Feedser.Normalization.csproj" /> + <ProjectReference Include="../StellaOps.Feedser.Storage.Mongo/StellaOps.Feedser.Storage.Mongo.csproj" /> + </ItemGroup> +</Project> diff --git a/src/StellaOps.Feedser.Source.Distro.Ubuntu/TASKS.md b/src/StellaOps.Feedser.Source.Distro.Ubuntu/TASKS.md new file mode 100644 index 00000000..d03dcb6f --- /dev/null +++ b/src/StellaOps.Feedser.Source.Distro.Ubuntu/TASKS.md @@ -0,0 +1,9 @@ +# Ubuntu Connector TODOs + +| Task | Status | Notes | +|---|---|---| +|Discover data model & pagination for `notices.json`|DONE|Connector now walks `offset`/`limit` pages (configurable page size) until MaxNoticesPerFetch satisfied, reusing cached pages when unchanged.| +|Design cursor & state model|DONE|Cursor tracks last published timestamp plus processed USN identifiers with overlap logic.| +|Implement fetch/parse pipeline|DONE|Index fetch hydrates per-notice DTOs, stores metadata, and maps without dedicated detail fetches.| +|Emit RangePrimitives + telemetry|DONE|Each package emits EVR primitives with `ubuntu.release` and `ubuntu.pocket` extensions for dashboards.| +|Add integration tests|DONE|Fixture-driven fetch→map suite covers resolved and ESM pockets, including conditional GET behaviour.| diff --git a/src/StellaOps.Feedser.Source.Distro.Ubuntu/UbuntuConnector.cs b/src/StellaOps.Feedser.Source.Distro.Ubuntu/UbuntuConnector.cs new file mode 100644 index 00000000..150e4458 --- /dev/null +++ b/src/StellaOps.Feedser.Source.Distro.Ubuntu/UbuntuConnector.cs @@ -0,0 +1,537 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Globalization; +using System.Text; +using System.Security.Cryptography; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Options; +using MongoDB.Bson; +using StellaOps.Feedser.Models; +using StellaOps.Feedser.Source.Common; +using StellaOps.Feedser.Source.Common.Fetch; +using StellaOps.Feedser.Source.Distro.Ubuntu.Configuration; +using StellaOps.Feedser.Source.Distro.Ubuntu.Internal; +using StellaOps.Feedser.Storage.Mongo; +using StellaOps.Feedser.Storage.Mongo.Advisories; +using StellaOps.Feedser.Storage.Mongo.Documents; +using StellaOps.Feedser.Storage.Mongo.Dtos; +using StellaOps.Plugin; + +namespace StellaOps.Feedser.Source.Distro.Ubuntu; + +public sealed class UbuntuConnector : IFeedConnector +{ + private readonly SourceFetchService _fetchService; + private readonly RawDocumentStorage _rawDocumentStorage; + private readonly IDocumentStore _documentStore; + private readonly IDtoStore _dtoStore; + private readonly IAdvisoryStore _advisoryStore; + private readonly ISourceStateRepository _stateRepository; + private readonly UbuntuOptions _options; + private readonly TimeProvider _timeProvider; + private readonly ILogger<UbuntuConnector> _logger; + + private static readonly Action<ILogger, string, int, Exception?> LogMapped = + LoggerMessage.Define<string, int>( + LogLevel.Information, + new EventId(1, "UbuntuMapped"), + "Ubuntu notice {NoticeId} mapped with {PackageCount} packages"); + + public UbuntuConnector( + SourceFetchService fetchService, + RawDocumentStorage rawDocumentStorage, + IDocumentStore documentStore, + IDtoStore dtoStore, + IAdvisoryStore advisoryStore, + ISourceStateRepository stateRepository, + IOptions<UbuntuOptions> options, + TimeProvider? timeProvider, + ILogger<UbuntuConnector> logger) + { + _fetchService = fetchService ?? throw new ArgumentNullException(nameof(fetchService)); + _rawDocumentStorage = rawDocumentStorage ?? throw new ArgumentNullException(nameof(rawDocumentStorage)); + _documentStore = documentStore ?? throw new ArgumentNullException(nameof(documentStore)); + _dtoStore = dtoStore ?? throw new ArgumentNullException(nameof(dtoStore)); + _advisoryStore = advisoryStore ?? throw new ArgumentNullException(nameof(advisoryStore)); + _stateRepository = stateRepository ?? throw new ArgumentNullException(nameof(stateRepository)); + _options = (options ?? throw new ArgumentNullException(nameof(options))).Value ?? throw new ArgumentNullException(nameof(options)); + _options.Validate(); + _timeProvider = timeProvider ?? TimeProvider.System; + _logger = logger ?? throw new ArgumentNullException(nameof(logger)); + } + + public string SourceName => UbuntuConnectorPlugin.SourceName; + + public async Task FetchAsync(IServiceProvider services, CancellationToken cancellationToken) + { + ArgumentNullException.ThrowIfNull(services); + + var cursor = await GetCursorAsync(cancellationToken).ConfigureAwait(false); + var now = _timeProvider.GetUtcNow(); + + var fetchCache = new Dictionary<string, UbuntuFetchCacheEntry>(cursor.FetchCache, StringComparer.OrdinalIgnoreCase); + var pendingMappings = new HashSet<Guid>(cursor.PendingMappings); + var processedIds = new HashSet<string>(cursor.ProcessedNoticeIds, StringComparer.OrdinalIgnoreCase); + + var indexResult = await FetchIndexAsync(cursor, fetchCache, now, cancellationToken).ConfigureAwait(false); + + if (indexResult.IsUnchanged) + { + await UpdateCursorAsync(cursor.WithFetchCache(fetchCache), cancellationToken).ConfigureAwait(false); + return; + } + + if (indexResult.Notices.Count == 0) + { + await UpdateCursorAsync(cursor.WithFetchCache(fetchCache), cancellationToken).ConfigureAwait(false); + return; + } + + var notices = indexResult.Notices; + + var baseline = (cursor.LastPublished ?? (now - _options.InitialBackfill)) - _options.ResumeOverlap; + if (baseline < DateTimeOffset.UnixEpoch) + { + baseline = DateTimeOffset.UnixEpoch; + } + + ProvenanceDiagnostics.ReportResumeWindow(SourceName, baseline, _logger); + + var candidates = notices + .Where(notice => notice.Published >= baseline) + .OrderBy(notice => notice.Published) + .ThenBy(notice => notice.NoticeId, StringComparer.OrdinalIgnoreCase) + .ToList(); + + if (candidates.Count == 0) + { + candidates = notices + .OrderByDescending(notice => notice.Published) + .ThenBy(notice => notice.NoticeId, StringComparer.OrdinalIgnoreCase) + .Take(_options.MaxNoticesPerFetch) + .OrderBy(notice => notice.Published) + .ThenBy(notice => notice.NoticeId, StringComparer.OrdinalIgnoreCase) + .ToList(); + } + else if (candidates.Count > _options.MaxNoticesPerFetch) + { + candidates = candidates + .OrderByDescending(notice => notice.Published) + .ThenBy(notice => notice.NoticeId, StringComparer.OrdinalIgnoreCase) + .Take(_options.MaxNoticesPerFetch) + .OrderBy(notice => notice.Published) + .ThenBy(notice => notice.NoticeId, StringComparer.OrdinalIgnoreCase) + .ToList(); + } + + var maxPublished = cursor.LastPublished ?? DateTimeOffset.MinValue; + var processedWindow = new List<string>(candidates.Count); + + foreach (var notice in candidates) + { + cancellationToken.ThrowIfCancellationRequested(); + + var detailUri = new Uri(_options.NoticeDetailBaseUri, notice.NoticeId); + var existing = await _documentStore.FindBySourceAndUriAsync(SourceName, detailUri.AbsoluteUri, cancellationToken).ConfigureAwait(false); + + var metadata = new Dictionary<string, string>(StringComparer.Ordinal) + { + ["ubuntu.id"] = notice.NoticeId, + ["ubuntu.published"] = notice.Published.ToString("O") + }; + + var dtoDocument = ToBson(notice); + var sha256 = ComputeNoticeHash(dtoDocument); + + var documentId = existing?.Id ?? Guid.NewGuid(); + var record = new DocumentRecord( + documentId, + SourceName, + detailUri.AbsoluteUri, + now, + sha256, + DocumentStatuses.PendingMap, + "application/json", + Headers: null, + Metadata: metadata, + Etag: existing?.Etag, + LastModified: existing?.LastModified ?? notice.Published, + GridFsId: null); + + await _documentStore.UpsertAsync(record, cancellationToken).ConfigureAwait(false); + + var dtoRecord = new DtoRecord(Guid.NewGuid(), record.Id, SourceName, "ubuntu.notice.v1", dtoDocument, now); + await _dtoStore.UpsertAsync(dtoRecord, cancellationToken).ConfigureAwait(false); + + pendingMappings.Add(record.Id); + processedIds.Add(notice.NoticeId); + processedWindow.Add(notice.NoticeId); + + if (notice.Published > maxPublished) + { + maxPublished = notice.Published; + } + } + + var updatedCursor = cursor + .WithFetchCache(fetchCache) + .WithPendingDocuments(Array.Empty<Guid>()) + .WithPendingMappings(pendingMappings) + .WithProcessed(maxPublished, processedWindow); + + await UpdateCursorAsync(updatedCursor, cancellationToken).ConfigureAwait(false); + } + + public Task ParseAsync(IServiceProvider services, CancellationToken cancellationToken) + => Task.CompletedTask; + + public async Task MapAsync(IServiceProvider services, CancellationToken cancellationToken) + { + ArgumentNullException.ThrowIfNull(services); + + var cursor = await GetCursorAsync(cancellationToken).ConfigureAwait(false); + if (cursor.PendingMappings.Count == 0) + { + return; + } + + var pending = cursor.PendingMappings.ToList(); + + foreach (var documentId in cursor.PendingMappings) + { + cancellationToken.ThrowIfCancellationRequested(); + + var dto = await _dtoStore.FindByDocumentIdAsync(documentId, cancellationToken).ConfigureAwait(false); + var document = await _documentStore.FindAsync(documentId, cancellationToken).ConfigureAwait(false); + + if (dto is null || document is null) + { + pending.Remove(documentId); + continue; + } + + UbuntuNoticeDto notice; + try + { + notice = FromBson(dto.Payload); + } + catch (Exception ex) + { + _logger.LogError(ex, "Failed to deserialize Ubuntu notice DTO for document {DocumentId}", documentId); + await _documentStore.UpdateStatusAsync(documentId, DocumentStatuses.Failed, cancellationToken).ConfigureAwait(false); + pending.Remove(documentId); + continue; + } + + var advisory = UbuntuMapper.Map(notice, document, _timeProvider.GetUtcNow()); + await _advisoryStore.UpsertAsync(advisory, cancellationToken).ConfigureAwait(false); + await _documentStore.UpdateStatusAsync(document.Id, DocumentStatuses.Mapped, cancellationToken).ConfigureAwait(false); + pending.Remove(documentId); + + LogMapped(_logger, notice.NoticeId, advisory.AffectedPackages.Length, null); + } + + var updatedCursor = cursor.WithPendingMappings(pending); + await UpdateCursorAsync(updatedCursor, cancellationToken).ConfigureAwait(false); + } + + private async Task<UbuntuIndexFetchResult> FetchIndexAsync( + UbuntuCursor cursor, + IDictionary<string, UbuntuFetchCacheEntry> fetchCache, + DateTimeOffset now, + CancellationToken cancellationToken) + { + var pageSize = Math.Clamp(_options.IndexPageSize, 1, UbuntuOptions.MaxPageSize); + var maxNotices = Math.Clamp(_options.MaxNoticesPerFetch, 1, 200); + var maxPages = Math.Max(1, (int)Math.Ceiling(maxNotices / (double)pageSize)); + var aggregated = new List<UbuntuNoticeDto>(Math.Min(maxNotices, pageSize * maxPages)); + var seenNoticeIds = new HashSet<string>(StringComparer.OrdinalIgnoreCase); + + var offset = 0; + var totalResults = int.MaxValue; + + for (var pageIndex = 0; pageIndex < maxPages && offset < totalResults; pageIndex++) + { + var pageUri = BuildIndexUri(_options.NoticesEndpoint, offset, pageSize); + var cacheKey = pageUri.ToString(); + + cursor.TryGetCache(cacheKey, out var cachedEntry); + + var metadata = new Dictionary<string, string>(StringComparer.Ordinal) + { + ["ubuntu.type"] = "index", + ["ubuntu.offset"] = offset.ToString(CultureInfo.InvariantCulture), + ["ubuntu.limit"] = pageSize.ToString(CultureInfo.InvariantCulture) + }; + + var indexRequest = new SourceFetchRequest(UbuntuOptions.HttpClientName, SourceName, pageUri) + { + Metadata = metadata, + ETag = cachedEntry?.ETag, + LastModified = cachedEntry?.LastModified, + TimeoutOverride = _options.FetchTimeout, + AcceptHeaders = new[] { "application/json" } + }; + + SourceFetchResult fetchResult; + try + { + fetchResult = await _fetchService.FetchAsync(indexRequest, cancellationToken).ConfigureAwait(false); + } + catch (Exception ex) + { + _logger.LogError(ex, "Ubuntu notices index fetch failed for {Uri}", pageUri); + await _stateRepository.MarkFailureAsync(SourceName, now, TimeSpan.FromMinutes(5), ex.Message, cancellationToken).ConfigureAwait(false); + throw; + } + + byte[] payload; + + if (fetchResult.IsNotModified) + { + if (pageIndex == 0) + { + if (cursor.FetchCache.TryGetValue(cacheKey, out var existingCache)) + { + fetchCache[cacheKey] = existingCache; + } + + return UbuntuIndexFetchResult.Unchanged(); + } + + if (!cursor.FetchCache.TryGetValue(cacheKey, out var cachedEntryForPage)) + { + break; + } + + fetchCache[cacheKey] = cachedEntryForPage; + + var existingDocument = await _documentStore.FindBySourceAndUriAsync(SourceName, cacheKey, cancellationToken).ConfigureAwait(false); + if (existingDocument is null || !existingDocument.GridFsId.HasValue) + { + break; + } + + payload = await _rawDocumentStorage.DownloadAsync(existingDocument.GridFsId.Value, cancellationToken).ConfigureAwait(false); + } + else + { + if (!fetchResult.IsSuccess || fetchResult.Document is null) + { + continue; + } + + fetchCache[cacheKey] = UbuntuFetchCacheEntry.FromDocument(fetchResult.Document); + + if (!fetchResult.Document.GridFsId.HasValue) + { + _logger.LogWarning("Ubuntu index document {DocumentId} missing GridFS payload", fetchResult.Document.Id); + continue; + } + + payload = await _rawDocumentStorage.DownloadAsync(fetchResult.Document.GridFsId.Value, cancellationToken).ConfigureAwait(false); + } + + var page = UbuntuNoticeParser.ParseIndex(Encoding.UTF8.GetString(payload)); + + if (page.TotalResults > 0) + { + totalResults = page.TotalResults; + } + + foreach (var notice in page.Notices) + { + if (!seenNoticeIds.Add(notice.NoticeId)) + { + continue; + } + + aggregated.Add(notice); + if (aggregated.Count >= maxNotices) + { + break; + } + } + + if (aggregated.Count >= maxNotices) + { + break; + } + + if (page.Notices.Count < pageSize) + { + break; + } + + offset += pageSize; + } + + return new UbuntuIndexFetchResult(false, aggregated); + } + + private static Uri BuildIndexUri(Uri endpoint, int offset, int limit) + { + var builder = new UriBuilder(endpoint); + var queryBuilder = new StringBuilder(); + + if (!string.IsNullOrEmpty(builder.Query)) + { + var existing = builder.Query.TrimStart('?'); + if (!string.IsNullOrEmpty(existing)) + { + queryBuilder.Append(existing); + if (existing[^1] != '&') + { + queryBuilder.Append('&'); + } + } + } + + queryBuilder.Append("offset="); + queryBuilder.Append(offset.ToString(CultureInfo.InvariantCulture)); + queryBuilder.Append("&limit="); + queryBuilder.Append(limit.ToString(CultureInfo.InvariantCulture)); + + builder.Query = queryBuilder.ToString(); + return builder.Uri; + } + + private sealed record UbuntuIndexFetchResult(bool IsUnchanged, IReadOnlyList<UbuntuNoticeDto> Notices) + { + public static UbuntuIndexFetchResult Unchanged() + => new(true, Array.Empty<UbuntuNoticeDto>()); + } + + private async Task<UbuntuCursor> GetCursorAsync(CancellationToken cancellationToken) + { + var state = await _stateRepository.TryGetAsync(SourceName, cancellationToken).ConfigureAwait(false); + return state is null ? UbuntuCursor.Empty : UbuntuCursor.FromBson(state.Cursor); + } + + private async Task UpdateCursorAsync(UbuntuCursor cursor, CancellationToken cancellationToken) + { + var doc = cursor.ToBsonDocument(); + await _stateRepository.UpdateCursorAsync(SourceName, doc, _timeProvider.GetUtcNow(), cancellationToken).ConfigureAwait(false); + } + + private static string ComputeNoticeHash(BsonDocument document) + { + var bytes = document.ToBson(); + var hash = SHA256.HashData(bytes); + return Convert.ToHexString(hash).ToLowerInvariant(); + } + + private static BsonDocument ToBson(UbuntuNoticeDto notice) + { + var packages = new BsonArray(); + foreach (var package in notice.Packages) + { + packages.Add(new BsonDocument + { + ["release"] = package.Release, + ["package"] = package.Package, + ["version"] = package.Version, + ["pocket"] = package.Pocket, + ["isSource"] = package.IsSource + }); + } + + var references = new BsonArray(); + foreach (var reference in notice.References) + { + var doc = new BsonDocument + { + ["url"] = reference.Url + }; + + if (!string.IsNullOrWhiteSpace(reference.Kind)) + { + doc["kind"] = reference.Kind; + } + + if (!string.IsNullOrWhiteSpace(reference.Title)) + { + doc["title"] = reference.Title; + } + + references.Add(doc); + } + + return new BsonDocument + { + ["noticeId"] = notice.NoticeId, + ["published"] = notice.Published.UtcDateTime, + ["title"] = notice.Title, + ["summary"] = notice.Summary, + ["cves"] = new BsonArray(notice.CveIds ?? Array.Empty<string>()), + ["packages"] = packages, + ["references"] = references + }; + } + + private static UbuntuNoticeDto FromBson(BsonDocument document) + { + var noticeId = document.GetValue("noticeId", string.Empty).AsString; + var published = document.TryGetValue("published", out var publishedValue) + ? publishedValue.BsonType switch + { + BsonType.DateTime => DateTime.SpecifyKind(publishedValue.ToUniversalTime(), DateTimeKind.Utc), + BsonType.String when DateTimeOffset.TryParse(publishedValue.AsString, out var parsed) => parsed.ToUniversalTime(), + _ => DateTimeOffset.UtcNow + } + : DateTimeOffset.UtcNow; + + var title = document.GetValue("title", noticeId).AsString; + var summary = document.GetValue("summary", string.Empty).AsString; + + var cves = document.TryGetValue("cves", out var cveArray) && cveArray is BsonArray cveBson + ? cveBson.OfType<BsonValue>() + .Select(static value => value?.ToString()) + .Where(static value => !string.IsNullOrWhiteSpace(value)) + .Select(static value => value!) + .ToArray() + : Array.Empty<string>(); + + var packages = new List<UbuntuReleasePackageDto>(); + if (document.TryGetValue("packages", out var packageArray) && packageArray is BsonArray packageBson) + { + foreach (var element in packageBson.OfType<BsonDocument>()) + { + packages.Add(new UbuntuReleasePackageDto( + Release: element.GetValue("release", string.Empty).AsString, + Package: element.GetValue("package", string.Empty).AsString, + Version: element.GetValue("version", string.Empty).AsString, + Pocket: element.GetValue("pocket", string.Empty).AsString, + IsSource: element.TryGetValue("isSource", out var sourceValue) && sourceValue.AsBoolean)); + } + } + + var references = new List<UbuntuReferenceDto>(); + if (document.TryGetValue("references", out var referenceArray) && referenceArray is BsonArray referenceBson) + { + foreach (var element in referenceBson.OfType<BsonDocument>()) + { + var url = element.GetValue("url", string.Empty).AsString; + if (string.IsNullOrWhiteSpace(url)) + { + continue; + } + + references.Add(new UbuntuReferenceDto( + url, + element.TryGetValue("kind", out var kindValue) ? kindValue.AsString : null, + element.TryGetValue("title", out var titleValue) ? titleValue.AsString : null)); + } + } + + return new UbuntuNoticeDto( + noticeId, + published, + title, + summary, + cves, + packages, + references); + } +} diff --git a/src/StellaOps.Feedser.Source.Distro.Ubuntu/UbuntuConnectorPlugin.cs b/src/StellaOps.Feedser.Source.Distro.Ubuntu/UbuntuConnectorPlugin.cs new file mode 100644 index 00000000..12a80e6c --- /dev/null +++ b/src/StellaOps.Feedser.Source.Distro.Ubuntu/UbuntuConnectorPlugin.cs @@ -0,0 +1,20 @@ +using System; +using Microsoft.Extensions.DependencyInjection; +using StellaOps.Plugin; + +namespace StellaOps.Feedser.Source.Distro.Ubuntu; + +public sealed class UbuntuConnectorPlugin : IConnectorPlugin +{ + public const string SourceName = "distro-ubuntu"; + + public string Name => SourceName; + + public bool IsAvailable(IServiceProvider services) => services is not null; + + public IFeedConnector Create(IServiceProvider services) + { + ArgumentNullException.ThrowIfNull(services); + return ActivatorUtilities.CreateInstance<UbuntuConnector>(services); + } +} diff --git a/src/StellaOps.Feedser.Source.Distro.Ubuntu/UbuntuDependencyInjectionRoutine.cs b/src/StellaOps.Feedser.Source.Distro.Ubuntu/UbuntuDependencyInjectionRoutine.cs new file mode 100644 index 00000000..07a4b704 --- /dev/null +++ b/src/StellaOps.Feedser.Source.Distro.Ubuntu/UbuntuDependencyInjectionRoutine.cs @@ -0,0 +1,53 @@ +using System; +using Microsoft.Extensions.Configuration; +using Microsoft.Extensions.DependencyInjection; +using StellaOps.DependencyInjection; +using StellaOps.Feedser.Core.Jobs; +using StellaOps.Feedser.Source.Distro.Ubuntu.Configuration; + +namespace StellaOps.Feedser.Source.Distro.Ubuntu; + +public sealed class UbuntuDependencyInjectionRoutine : IDependencyInjectionRoutine +{ + private const string ConfigurationSection = "feedser:sources:ubuntu"; + private const string FetchCron = "*/20 * * * *"; + private const string ParseCron = "7,27,47 * * * *"; + private const string MapCron = "10,30,50 * * * *"; + + private static readonly TimeSpan FetchTimeout = TimeSpan.FromMinutes(4); + private static readonly TimeSpan ParseTimeout = TimeSpan.FromMinutes(5); + private static readonly TimeSpan MapTimeout = TimeSpan.FromMinutes(8); + private static readonly TimeSpan LeaseDuration = TimeSpan.FromMinutes(3); + + public IServiceCollection Register(IServiceCollection services, IConfiguration configuration) + { + ArgumentNullException.ThrowIfNull(services); + ArgumentNullException.ThrowIfNull(configuration); + + services.AddUbuntuConnector(options => + { + configuration.GetSection(ConfigurationSection).Bind(options); + options.Validate(); + }); + + var scheduler = new JobSchedulerBuilder(services); + scheduler + .AddJob<UbuntuFetchJob>( + UbuntuJobKinds.Fetch, + cronExpression: FetchCron, + timeout: FetchTimeout, + leaseDuration: LeaseDuration) + .AddJob<UbuntuParseJob>( + UbuntuJobKinds.Parse, + cronExpression: ParseCron, + timeout: ParseTimeout, + leaseDuration: LeaseDuration) + .AddJob<UbuntuMapJob>( + UbuntuJobKinds.Map, + cronExpression: MapCron, + timeout: MapTimeout, + leaseDuration: LeaseDuration); + + return services; + } +} diff --git a/src/StellaOps.Feedser.Source.Distro.Ubuntu/UbuntuServiceCollectionExtensions.cs b/src/StellaOps.Feedser.Source.Distro.Ubuntu/UbuntuServiceCollectionExtensions.cs new file mode 100644 index 00000000..ff362f3d --- /dev/null +++ b/src/StellaOps.Feedser.Source.Distro.Ubuntu/UbuntuServiceCollectionExtensions.cs @@ -0,0 +1,37 @@ +using System; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Options; +using StellaOps.Feedser.Source.Common.Http; +using StellaOps.Feedser.Source.Distro.Ubuntu.Configuration; + +namespace StellaOps.Feedser.Source.Distro.Ubuntu; + +public static class UbuntuServiceCollectionExtensions +{ + public static IServiceCollection AddUbuntuConnector(this IServiceCollection services, Action<UbuntuOptions> configure) + { + ArgumentNullException.ThrowIfNull(services); + ArgumentNullException.ThrowIfNull(configure); + + services.AddOptions<UbuntuOptions>() + .Configure(configure) + .PostConfigure(static options => options.Validate()); + + services.AddSourceHttpClient(UbuntuOptions.HttpClientName, (sp, httpOptions) => + { + var options = sp.GetRequiredService<IOptions<UbuntuOptions>>().Value; + httpOptions.BaseAddress = options.NoticesEndpoint.GetLeftPart(UriPartial.Authority) is { Length: > 0 } authority + ? new Uri(authority) + : new Uri("https://ubuntu.com/"); + httpOptions.Timeout = options.FetchTimeout; + httpOptions.UserAgent = options.UserAgent; + httpOptions.AllowedHosts.Clear(); + httpOptions.AllowedHosts.Add(options.NoticesEndpoint.Host); + httpOptions.AllowedHosts.Add(options.NoticeDetailBaseUri.Host); + httpOptions.DefaultRequestHeaders["Accept"] = "application/json"; + }); + + services.AddTransient<UbuntuConnector>(); + return services; + } +} diff --git a/src/StellaOps.Feedser.Source.Ghsa.Tests/Fixtures/expected-GHSA-xxxx-yyyy-zzzz.json b/src/StellaOps.Feedser.Source.Ghsa.Tests/Fixtures/expected-GHSA-xxxx-yyyy-zzzz.json new file mode 100644 index 00000000..5c81b133 --- /dev/null +++ b/src/StellaOps.Feedser.Source.Ghsa.Tests/Fixtures/expected-GHSA-xxxx-yyyy-zzzz.json @@ -0,0 +1,113 @@ +{ + "advisoryKey": "GHSA-xxxx-yyyy-zzzz", + "affectedPackages": [ + { + "identifier": "npm:example/package", + "platform": null, + "provenance": [ + { + "fieldMask": [], + "kind": "affected", + "recordedAt": "2024-10-02T00:00:00+00:00", + "source": "ghsa", + "value": "npm:example/package" + } + ], + "statuses": [ + { + "provenance": { + "fieldMask": [], + "kind": "affected", + "recordedAt": "2024-10-02T00:00:00+00:00", + "source": "ghsa", + "value": "npm:example/package" + }, + "status": "affected" + } + ], + "type": "semver", + "versionRanges": [ + { + "fixedVersion": "1.5.0", + "introducedVersion": null, + "lastAffectedVersion": null, + "primitives": { + "evr": null, + "hasVendorExtensions": true, + "nevra": null, + "semVer": null, + "vendorExtensions": { + "ecosystem": "npm", + "package": "example/package" + } + }, + "provenance": { + "fieldMask": [], + "kind": "affected", + "recordedAt": "2024-10-02T00:00:00+00:00", + "source": "ghsa", + "value": "npm:example/package" + }, + "rangeExpression": "< 1.5.0", + "rangeKind": "semver" + } + ] + } + ], + "aliases": [ + "CVE-2024-1111", + "GHSA-xxxx-yyyy-zzzz" + ], + "cvssMetrics": [], + "exploitKnown": false, + "language": "en", + "modified": "2024-09-20T12:00:00+00:00", + "provenance": [ + { + "fieldMask": [], + "kind": "document", + "recordedAt": "2024-10-02T00:00:00+00:00", + "source": "ghsa", + "value": "security/advisories/GHSA-xxxx-yyyy-zzzz" + }, + { + "fieldMask": [], + "kind": "mapping", + "recordedAt": "2024-10-02T00:00:00+00:00", + "source": "ghsa", + "value": "GHSA-xxxx-yyyy-zzzz" + } + ], + "published": "2024-09-10T13:00:00+00:00", + "references": [ + { + "kind": "fix", + "provenance": { + "fieldMask": [], + "kind": "reference", + "recordedAt": "2024-10-02T00:00:00+00:00", + "source": "ghsa", + "value": "https://example.com/patch" + }, + "sourceTag": "Vendor Fix", + "summary": null, + "url": "https://example.com/patch" + }, + { + "kind": "advisory", + "provenance": { + "fieldMask": [], + "kind": "reference", + "recordedAt": "2024-10-02T00:00:00+00:00", + "source": "ghsa", + "value": "https://github.com/advisories/GHSA-xxxx-yyyy-zzzz" + }, + "sourceTag": null, + "summary": null, + "url": "https://github.com/advisories/GHSA-xxxx-yyyy-zzzz" + } + ], + "severity": "critical", + "summary": "Example GHSA vulnerability", + "title": "Example GHSA vulnerability" +} diff --git a/src/StellaOps.Feedser.Source.Ghsa.Tests/Fixtures/ghsa-GHSA-xxxx-yyyy-zzzz.json b/src/StellaOps.Feedser.Source.Ghsa.Tests/Fixtures/ghsa-GHSA-xxxx-yyyy-zzzz.json new file mode 100644 index 00000000..70b7caf3 --- /dev/null +++ b/src/StellaOps.Feedser.Source.Ghsa.Tests/Fixtures/ghsa-GHSA-xxxx-yyyy-zzzz.json @@ -0,0 +1,50 @@ +{ + "ghsa_id": "GHSA-xxxx-yyyy-zzzz", + "summary": "Example GHSA vulnerability", + "description": "An example advisory describing a supply chain risk.", + "severity": "CRITICAL", + "published_at": "2024-09-10T13:00:00Z", + "updated_at": "2024-09-20T12:00:00Z", + "cve_ids": [ + "CVE-2024-1111" + ], + "references": [ + { + "url": "https://github.com/advisories/GHSA-xxxx-yyyy-zzzz", + "type": "ADVISORY" + }, + { + "url": "https://example.com/patch", + "type": "FIX", + "name": "Vendor Fix" + } + ], + "credits": [ + { + "type": "reporter", + "user": { + "login": "security-reporter", + "html_url": "https://github.com/security-reporter" + } + }, + { + "type": "remediation_developer", + "user": { + "login": "maintainer-team", + "html_url": "https://github.com/maintainer-team" + } + } + ], + "vulnerabilities": [ + { + "package": { + "name": "example/package", + "ecosystem": "npm" + }, + "vulnerable_version_range": "< 1.5.0", + "first_patched_version": { + "identifier": "1.5.0" + } + } + ] +} diff --git a/src/StellaOps.Feedser.Source.Ghsa.Tests/Fixtures/ghsa-list.json b/src/StellaOps.Feedser.Source.Ghsa.Tests/Fixtures/ghsa-list.json new file mode 100644 index 00000000..b73b82ae --- /dev/null +++ b/src/StellaOps.Feedser.Source.Ghsa.Tests/Fixtures/ghsa-list.json @@ -0,0 +1,12 @@ +{ + "advisories": [ + { + "ghsa_id": "GHSA-xxxx-yyyy-zzzz", + "updated_at": "2024-09-20T12:00:00Z" + } + ], + "pagination": { + "page": 1, + "has_next_page": false + } +} diff --git a/src/StellaOps.Feedser.Source.Ghsa.Tests/Ghsa/GhsaConnectorTests.cs b/src/StellaOps.Feedser.Source.Ghsa.Tests/Ghsa/GhsaConnectorTests.cs new file mode 100644 index 00000000..b3d39e52 --- /dev/null +++ b/src/StellaOps.Feedser.Source.Ghsa.Tests/Ghsa/GhsaConnectorTests.cs @@ -0,0 +1,135 @@ +using System.Net; +using System.Net.Http; +using System.Text; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Logging.Abstractions; +using StellaOps.Feedser.Models; +using StellaOps.Feedser.Source.Common.Testing; +using StellaOps.Feedser.Source.Ghsa.Configuration; +using StellaOps.Feedser.Testing; +using StellaOps.Feedser.Storage.Mongo.Advisories; + +namespace StellaOps.Feedser.Source.Ghsa.Tests; + +[Collection("mongo-fixture")] +public sealed class GhsaConnectorTests : IAsyncLifetime +{ + private readonly MongoIntegrationFixture _fixture; + private ConnectorTestHarness? _harness; + + public GhsaConnectorTests(MongoIntegrationFixture fixture) + { + _fixture = fixture; + } + + [Fact] + public async Task FetchParseMap_EmitsCanonicalAdvisory() + { + var initialTime = new DateTimeOffset(2024, 10, 2, 0, 0, 0, TimeSpan.Zero); + await EnsureHarnessAsync(initialTime); + var harness = _harness!; + + var since = initialTime - TimeSpan.FromDays(30); + var listUri = new Uri($"https://ghsa.test/security/advisories?updated_since={Uri.EscapeDataString(since.ToString("O"))}&updated_until={Uri.EscapeDataString(initialTime.ToString("O"))}&page=1&per_page=5"); + harness.Handler.AddJsonResponse(listUri, ReadFixture("Fixtures/ghsa-list.json")); + harness.Handler.SetFallback(request => + { + if (request.RequestUri is null) + { + return new HttpResponseMessage(HttpStatusCode.NotFound); + } + + if (request.RequestUri.AbsoluteUri.Equals("https://ghsa.test/security/advisories/GHSA-xxxx-yyyy-zzzz", StringComparison.OrdinalIgnoreCase)) + { + return new HttpResponseMessage(HttpStatusCode.OK) + { + Content = new StringContent(ReadFixture("Fixtures/ghsa-GHSA-xxxx-yyyy-zzzz.json"), Encoding.UTF8, "application/json") + }; + } + + return new HttpResponseMessage(HttpStatusCode.NotFound); + }); + + var connector = new GhsaConnectorPlugin().Create(harness.ServiceProvider); + + await connector.FetchAsync(harness.ServiceProvider, CancellationToken.None); + await connector.ParseAsync(harness.ServiceProvider, CancellationToken.None); + await connector.MapAsync(harness.ServiceProvider, CancellationToken.None); + + var advisoryStore = harness.ServiceProvider.GetRequiredService<IAdvisoryStore>(); + var advisory = await advisoryStore.FindAsync("GHSA-xxxx-yyyy-zzzz", CancellationToken.None); + Assert.NotNull(advisory); + + Assert.Collection(advisory!.Credits, + credit => + { + Assert.Equal("remediation_developer", credit.Role); + Assert.Equal("maintainer-team", credit.DisplayName); + Assert.Contains("https://github.com/maintainer-team", credit.Contacts); + }, + credit => + { + Assert.Equal("reporter", credit.Role); + Assert.Equal("security-reporter", credit.DisplayName); + Assert.Contains("https://github.com/security-reporter", credit.Contacts); + }); + + var snapshot = SnapshotSerializer.ToSnapshot(advisory).Replace("\r\n", "\n").TrimEnd(); + var expected = ReadFixture("Fixtures/expected-GHSA-xxxx-yyyy-zzzz.json").Replace("\r\n", "\n").TrimEnd(); + + if (!string.Equals(expected, snapshot, StringComparison.Ordinal)) + { + var actualPath = Path.Combine(AppContext.BaseDirectory, "Fixtures", "expected-GHSA-xxxx-yyyy-zzzz.actual.json"); + Directory.CreateDirectory(Path.GetDirectoryName(actualPath)!); + File.WriteAllText(actualPath, snapshot); + } + + Assert.Equal(expected, snapshot); + harness.Handler.AssertNoPendingResponses(); + } + + private async Task EnsureHarnessAsync(DateTimeOffset initialTime) + { + if (_harness is not null) + { + return; + } + + var harness = new ConnectorTestHarness(_fixture, initialTime, GhsaOptions.HttpClientName); + await harness.EnsureServiceProviderAsync(services => + { + services.AddLogging(builder => builder.AddProvider(NullLoggerProvider.Instance)); + services.AddGhsaConnector(options => + { + options.BaseEndpoint = new Uri("https://ghsa.test/", UriKind.Absolute); + options.ApiToken = "test-token"; + options.PageSize = 5; + options.MaxPagesPerFetch = 2; + options.RequestDelay = TimeSpan.Zero; + options.InitialBackfill = TimeSpan.FromDays(30); + }); + }); + + _harness = harness; + } + + private static string ReadFixture(string relativePath) + { + var path = Path.Combine(AppContext.BaseDirectory, relativePath); + return File.ReadAllText(path); + } + + public async Task InitializeAsync() + { + await Task.CompletedTask; + } + + public async Task DisposeAsync() + { + if (_harness is not null) + { + await _harness.DisposeAsync(); + } + } +} diff --git a/src/StellaOps.Feedser.Source.Ghsa.Tests/StellaOps.Feedser.Source.Ghsa.Tests.csproj b/src/StellaOps.Feedser.Source.Ghsa.Tests/StellaOps.Feedser.Source.Ghsa.Tests.csproj new file mode 100644 index 00000000..c476b167 --- /dev/null +++ b/src/StellaOps.Feedser.Source.Ghsa.Tests/StellaOps.Feedser.Source.Ghsa.Tests.csproj @@ -0,0 +1,17 @@ +<Project Sdk="Microsoft.NET.Sdk"> + <PropertyGroup> + <TargetFramework>net10.0</TargetFramework> + <ImplicitUsings>enable</ImplicitUsings> + <Nullable>enable</Nullable> + </PropertyGroup> + <ItemGroup> + <ProjectReference Include="../StellaOps.Feedser.Source.Ghsa/StellaOps.Feedser.Source.Ghsa.csproj" /> + <ProjectReference Include="../StellaOps.Feedser.Source.Common/StellaOps.Feedser.Source.Common.csproj" /> + <ProjectReference Include="../StellaOps.Feedser.Models/StellaOps.Feedser.Models.csproj" /> + <ProjectReference Include="../StellaOps.Feedser.Storage.Mongo/StellaOps.Feedser.Storage.Mongo.csproj" /> + <ProjectReference Include="../StellaOps.Feedser.Testing/StellaOps.Feedser.Testing.csproj" /> + </ItemGroup> + <ItemGroup> + <None Include="Fixtures/*.json" CopyToOutputDirectory="Always" /> + </ItemGroup> +</Project> diff --git a/src/StellaOps.Feedser.Source.Ghsa/AGENTS.md b/src/StellaOps.Feedser.Source.Ghsa/AGENTS.md new file mode 100644 index 00000000..e80bd4c5 --- /dev/null +++ b/src/StellaOps.Feedser.Source.Ghsa/AGENTS.md @@ -0,0 +1,39 @@ +# AGENTS +## Role +Implement a connector for GitHub Security Advisories (GHSA) when we need to ingest GHSA content directly (instead of crosswalking via OSV/NVD). + +## Scope +- Determine the optimal GHSA data source (GraphQL API, REST, or ecosystem export) and required authentication. +- Implement fetch logic with pagination, updated-since filtering, and cursor persistence. +- Parse GHSA records (identifiers, summaries, affected packages, versions, references, severity). +- Map advisories into canonical `Advisory` objects with aliases, references, affected packages, and range primitives. +- Provide deterministic fixtures and regression tests for the full pipeline. + +## Participants +- `Source.Common` (HTTP clients, fetch service, DTO storage). +- `Storage.Mongo` (raw/document/DTO/advisory stores and source state). +- `Feedser.Models` (canonical advisory types). +- `Feedser.Testing` (integration harness, snapshot helpers). + +## Interfaces & Contracts +- Job kinds: `ghsa:fetch`, `ghsa:parse`, `ghsa:map`. +- Support GitHub API authentication & rate limiting (token, retry/backoff). +- Alias set must include GHSA IDs and linked CVE IDs. + +## In/Out of scope +In scope: +- Full GHSA connector implementation with range primitives and provenance instrumentation. + +Out of scope: +- Repo-specific advisory ingest (handled via GitHub repo exports). +- Downstream ecosystem-specific enrichments. + +## Observability & Security Expectations +- Log fetch pagination, throttling, and mapping stats. +- Handle GitHub API rate limits with exponential backoff and `Retry-After`. +- Sanitize/validate payloads before persistence. + +## Tests +- Add `StellaOps.Feedser.Source.Ghsa.Tests` with canned GraphQL/REST fixtures. +- Snapshot canonical advisories; enable fixture regeneration with env flag. +- Confirm deterministic ordering/time normalisation. diff --git a/src/StellaOps.Feedser.Source.Ghsa/Configuration/GhsaOptions.cs b/src/StellaOps.Feedser.Source.Ghsa/Configuration/GhsaOptions.cs new file mode 100644 index 00000000..3b333c2b --- /dev/null +++ b/src/StellaOps.Feedser.Source.Ghsa/Configuration/GhsaOptions.cs @@ -0,0 +1,61 @@ +using System.Diagnostics.CodeAnalysis; + +namespace StellaOps.Feedser.Source.Ghsa.Configuration; + +public sealed class GhsaOptions +{ + public static string HttpClientName => "source.ghsa"; + + public Uri BaseEndpoint { get; set; } = new("https://api.github.com/", UriKind.Absolute); + + public string ApiToken { get; set; } = string.Empty; + + public int PageSize { get; set; } = 50; + + public int MaxPagesPerFetch { get; set; } = 5; + + public TimeSpan InitialBackfill { get; set; } = TimeSpan.FromDays(30); + + public TimeSpan RequestDelay { get; set; } = TimeSpan.FromMilliseconds(200); + + public TimeSpan FailureBackoff { get; set; } = TimeSpan.FromMinutes(5); + + [MemberNotNull(nameof(BaseEndpoint), nameof(ApiToken))] + public void Validate() + { + if (BaseEndpoint is null || !BaseEndpoint.IsAbsoluteUri) + { + throw new InvalidOperationException("BaseEndpoint must be an absolute URI."); + } + + if (string.IsNullOrWhiteSpace(ApiToken)) + { + throw new InvalidOperationException("ApiToken must be provided."); + } + + if (PageSize is < 1 or > 100) + { + throw new InvalidOperationException("PageSize must be between 1 and 100."); + } + + if (MaxPagesPerFetch <= 0) + { + throw new InvalidOperationException("MaxPagesPerFetch must be positive."); + } + + if (InitialBackfill < TimeSpan.Zero) + { + throw new InvalidOperationException("InitialBackfill cannot be negative."); + } + + if (RequestDelay < TimeSpan.Zero) + { + throw new InvalidOperationException("RequestDelay cannot be negative."); + } + + if (FailureBackoff <= TimeSpan.Zero) + { + throw new InvalidOperationException("FailureBackoff must be greater than zero."); + } + } +} diff --git a/src/StellaOps.Feedser.Source.Ghsa/GhsaConnector.cs b/src/StellaOps.Feedser.Source.Ghsa/GhsaConnector.cs new file mode 100644 index 00000000..d3fae88c --- /dev/null +++ b/src/StellaOps.Feedser.Source.Ghsa/GhsaConnector.cs @@ -0,0 +1,394 @@ +using System.Collections.Generic; +using System.Globalization; +using System.Linq; +using System.Net.Http; +using System.Text.Json; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Options; +using MongoDB.Bson; +using StellaOps.Feedser.Models; +using StellaOps.Feedser.Source.Common; +using StellaOps.Feedser.Source.Common.Fetch; +using StellaOps.Feedser.Source.Ghsa.Configuration; +using StellaOps.Feedser.Source.Ghsa.Internal; +using StellaOps.Feedser.Storage.Mongo; +using StellaOps.Feedser.Storage.Mongo.Advisories; +using StellaOps.Feedser.Storage.Mongo.Documents; +using StellaOps.Feedser.Storage.Mongo.Dtos; +using StellaOps.Plugin; + +namespace StellaOps.Feedser.Source.Ghsa; + +public sealed class GhsaConnector : IFeedConnector +{ + private static readonly JsonSerializerOptions SerializerOptions = new(JsonSerializerDefaults.Web) + { + PropertyNameCaseInsensitive = true, + WriteIndented = false, + }; + + private readonly SourceFetchService _fetchService; + private readonly RawDocumentStorage _rawDocumentStorage; + private readonly IDocumentStore _documentStore; + private readonly IDtoStore _dtoStore; + private readonly IAdvisoryStore _advisoryStore; + private readonly ISourceStateRepository _stateRepository; + private readonly GhsaOptions _options; + private readonly GhsaDiagnostics _diagnostics; + private readonly TimeProvider _timeProvider; + private readonly ILogger<GhsaConnector> _logger; + + public GhsaConnector( + SourceFetchService fetchService, + RawDocumentStorage rawDocumentStorage, + IDocumentStore documentStore, + IDtoStore dtoStore, + IAdvisoryStore advisoryStore, + ISourceStateRepository stateRepository, + IOptions<GhsaOptions> options, + GhsaDiagnostics diagnostics, + TimeProvider? timeProvider, + ILogger<GhsaConnector> logger) + { + _fetchService = fetchService ?? throw new ArgumentNullException(nameof(fetchService)); + _rawDocumentStorage = rawDocumentStorage ?? throw new ArgumentNullException(nameof(rawDocumentStorage)); + _documentStore = documentStore ?? throw new ArgumentNullException(nameof(documentStore)); + _dtoStore = dtoStore ?? throw new ArgumentNullException(nameof(dtoStore)); + _advisoryStore = advisoryStore ?? throw new ArgumentNullException(nameof(advisoryStore)); + _stateRepository = stateRepository ?? throw new ArgumentNullException(nameof(stateRepository)); + _options = (options ?? throw new ArgumentNullException(nameof(options))).Value ?? throw new ArgumentNullException(nameof(options)); + _options.Validate(); + _diagnostics = diagnostics ?? throw new ArgumentNullException(nameof(diagnostics)); + _timeProvider = timeProvider ?? TimeProvider.System; + _logger = logger ?? throw new ArgumentNullException(nameof(logger)); + } + + public string SourceName => GhsaConnectorPlugin.SourceName; + + public async Task FetchAsync(IServiceProvider services, CancellationToken cancellationToken) + { + ArgumentNullException.ThrowIfNull(services); + + var now = _timeProvider.GetUtcNow(); + var cursor = await GetCursorAsync(cancellationToken).ConfigureAwait(false); + + var pendingDocuments = cursor.PendingDocuments.ToHashSet(); + var pendingMappings = cursor.PendingMappings.ToHashSet(); + + var since = cursor.CurrentWindowStart ?? cursor.LastUpdatedExclusive ?? now - _options.InitialBackfill; + if (since > now) + { + since = now; + } + + var until = cursor.CurrentWindowEnd ?? now; + if (until <= since) + { + until = since + TimeSpan.FromMinutes(1); + } + + var page = cursor.NextPage <= 0 ? 1 : cursor.NextPage; + var pagesFetched = 0; + var hasMore = true; + DateTimeOffset? maxUpdated = cursor.LastUpdatedExclusive; + + while (hasMore && pagesFetched < _options.MaxPagesPerFetch) + { + cancellationToken.ThrowIfCancellationRequested(); + + var listUri = BuildListUri(since, until, page, _options.PageSize); + var metadata = new Dictionary<string, string>(StringComparer.Ordinal) + { + ["since"] = since.ToString("O"), + ["until"] = until.ToString("O"), + ["page"] = page.ToString(CultureInfo.InvariantCulture), + ["pageSize"] = _options.PageSize.ToString(CultureInfo.InvariantCulture), + }; + + SourceFetchContentResult listResult; + try + { + _diagnostics.FetchAttempt(); + listResult = await _fetchService.FetchContentAsync( + new SourceFetchRequest( + GhsaOptions.HttpClientName, + SourceName, + listUri) + { + Metadata = metadata, + AcceptHeaders = new[] { "application/vnd.github+json" }, + }, + cancellationToken).ConfigureAwait(false); + } + catch (HttpRequestException ex) + { + _diagnostics.FetchFailure(); + await _stateRepository.MarkFailureAsync(SourceName, now, _options.FailureBackoff, ex.Message, cancellationToken).ConfigureAwait(false); + throw; + } + + if (listResult.IsNotModified) + { + _diagnostics.FetchUnchanged(); + break; + } + + if (!listResult.IsSuccess || listResult.Content is null) + { + _diagnostics.FetchFailure(); + break; + } + + var pageModel = GhsaListParser.Parse(listResult.Content, page, _options.PageSize); + + if (pageModel.Items.Count == 0) + { + hasMore = false; + } + + foreach (var item in pageModel.Items) + { + cancellationToken.ThrowIfCancellationRequested(); + + var detailUri = BuildDetailUri(item.GhsaId); + var detailMetadata = new Dictionary<string, string>(StringComparer.Ordinal) + { + ["ghsaId"] = item.GhsaId, + ["page"] = page.ToString(CultureInfo.InvariantCulture), + ["since"] = since.ToString("O"), + ["until"] = until.ToString("O"), + }; + + SourceFetchResult detailResult; + try + { + detailResult = await _fetchService.FetchAsync( + new SourceFetchRequest( + GhsaOptions.HttpClientName, + SourceName, + detailUri) + { + Metadata = detailMetadata, + AcceptHeaders = new[] { "application/vnd.github+json" }, + }, + cancellationToken).ConfigureAwait(false); + } + catch (HttpRequestException ex) + { + _diagnostics.FetchFailure(); + _logger.LogWarning(ex, "Failed fetching GHSA advisory {GhsaId}", item.GhsaId); + continue; + } + + if (detailResult.IsNotModified) + { + _diagnostics.FetchUnchanged(); + continue; + } + + if (!detailResult.IsSuccess || detailResult.Document is null) + { + _diagnostics.FetchFailure(); + continue; + } + + _diagnostics.FetchDocument(); + pendingDocuments.Add(detailResult.Document.Id); + pendingMappings.Add(detailResult.Document.Id); + } + + if (pageModel.MaxUpdated.HasValue) + { + if (!maxUpdated.HasValue || pageModel.MaxUpdated > maxUpdated) + { + maxUpdated = pageModel.MaxUpdated; + } + } + + hasMore = pageModel.HasMorePages; + page = pageModel.NextPageCandidate; + pagesFetched++; + + if (hasMore && _options.RequestDelay > TimeSpan.Zero) + { + await Task.Delay(_options.RequestDelay, cancellationToken).ConfigureAwait(false); + } + } + + var updatedCursor = cursor + .WithPendingDocuments(pendingDocuments) + .WithPendingMappings(pendingMappings); + + if (hasMore) + { + updatedCursor = updatedCursor + .WithCurrentWindowStart(since) + .WithCurrentWindowEnd(until) + .WithNextPage(page); + } + else + { + var nextSince = maxUpdated ?? until; + updatedCursor = updatedCursor + .WithLastUpdatedExclusive(nextSince) + .WithCurrentWindowStart(null) + .WithCurrentWindowEnd(null) + .WithNextPage(1); + } + + await UpdateCursorAsync(updatedCursor, cancellationToken).ConfigureAwait(false); + } + + public async Task ParseAsync(IServiceProvider services, CancellationToken cancellationToken) + { + ArgumentNullException.ThrowIfNull(services); + + var cursor = await GetCursorAsync(cancellationToken).ConfigureAwait(false); + if (cursor.PendingDocuments.Count == 0) + { + return; + } + + var remainingDocuments = cursor.PendingDocuments.ToList(); + + foreach (var documentId in cursor.PendingDocuments) + { + cancellationToken.ThrowIfCancellationRequested(); + + var document = await _documentStore.FindAsync(documentId, cancellationToken).ConfigureAwait(false); + if (document is null) + { + remainingDocuments.Remove(documentId); + continue; + } + + if (!document.GridFsId.HasValue) + { + _diagnostics.ParseFailure(); + _logger.LogWarning("GHSA document {DocumentId} missing GridFS content", documentId); + await _documentStore.UpdateStatusAsync(document.Id, DocumentStatuses.Failed, cancellationToken).ConfigureAwait(false); + remainingDocuments.Remove(documentId); + continue; + } + + byte[] rawBytes; + try + { + rawBytes = await _rawDocumentStorage.DownloadAsync(document.GridFsId.Value, cancellationToken).ConfigureAwait(false); + } + catch (Exception ex) + { + _diagnostics.ParseFailure(); + _logger.LogError(ex, "Unable to download GHSA raw document {DocumentId}", documentId); + throw; + } + + GhsaRecordDto dto; + try + { + dto = GhsaRecordParser.Parse(rawBytes); + } + catch (JsonException ex) + { + _diagnostics.ParseQuarantine(); + _logger.LogError(ex, "Malformed GHSA JSON for {DocumentId}", documentId); + await _documentStore.UpdateStatusAsync(document.Id, DocumentStatuses.Failed, cancellationToken).ConfigureAwait(false); + remainingDocuments.Remove(documentId); + continue; + } + + var payload = BsonDocument.Parse(JsonSerializer.Serialize(dto, SerializerOptions)); + var dtoRecord = new DtoRecord( + Guid.NewGuid(), + document.Id, + SourceName, + "ghsa/1.0", + payload, + _timeProvider.GetUtcNow()); + + await _dtoStore.UpsertAsync(dtoRecord, cancellationToken).ConfigureAwait(false); + await _documentStore.UpdateStatusAsync(document.Id, DocumentStatuses.PendingMap, cancellationToken).ConfigureAwait(false); + + remainingDocuments.Remove(documentId); + _diagnostics.ParseSuccess(); + } + + var updatedCursor = cursor.WithPendingDocuments(remainingDocuments); + await UpdateCursorAsync(updatedCursor, cancellationToken).ConfigureAwait(false); + } + + public async Task MapAsync(IServiceProvider services, CancellationToken cancellationToken) + { + ArgumentNullException.ThrowIfNull(services); + + var cursor = await GetCursorAsync(cancellationToken).ConfigureAwait(false); + if (cursor.PendingMappings.Count == 0) + { + return; + } + + var pendingMappings = cursor.PendingMappings.ToList(); + + foreach (var documentId in cursor.PendingMappings) + { + cancellationToken.ThrowIfCancellationRequested(); + + var dtoRecord = await _dtoStore.FindByDocumentIdAsync(documentId, cancellationToken).ConfigureAwait(false); + var document = await _documentStore.FindAsync(documentId, cancellationToken).ConfigureAwait(false); + + if (dtoRecord is null || document is null) + { + _logger.LogWarning("Skipping GHSA mapping for {DocumentId}: DTO or document missing", documentId); + pendingMappings.Remove(documentId); + continue; + } + + GhsaRecordDto dto; + try + { + dto = JsonSerializer.Deserialize<GhsaRecordDto>(dtoRecord.Payload.ToJson(), SerializerOptions) + ?? throw new InvalidOperationException("Deserialized DTO was null."); + } + catch (Exception ex) + { + _logger.LogError(ex, "Failed to deserialize GHSA DTO for {DocumentId}", documentId); + await _documentStore.UpdateStatusAsync(document.Id, DocumentStatuses.Failed, cancellationToken).ConfigureAwait(false); + pendingMappings.Remove(documentId); + continue; + } + + var advisory = GhsaMapper.Map(dto, document, dtoRecord.ValidatedAt); + + await _advisoryStore.UpsertAsync(advisory, cancellationToken).ConfigureAwait(false); + await _documentStore.UpdateStatusAsync(document.Id, DocumentStatuses.Mapped, cancellationToken).ConfigureAwait(false); + pendingMappings.Remove(documentId); + _diagnostics.MapSuccess(1); + } + + var updatedCursor = cursor.WithPendingMappings(pendingMappings); + await UpdateCursorAsync(updatedCursor, cancellationToken).ConfigureAwait(false); + } + + private static Uri BuildListUri(DateTimeOffset since, DateTimeOffset until, int page, int pageSize) + { + var query = $"updated_since={Uri.EscapeDataString(since.ToString("O"))}&updated_until={Uri.EscapeDataString(until.ToString("O"))}&page={page}&per_page={pageSize}"; + return new Uri($"security/advisories?{query}", UriKind.Relative); + } + + private static Uri BuildDetailUri(string ghsaId) + { + var encoded = Uri.EscapeDataString(ghsaId); + return new Uri($"security/advisories/{encoded}", UriKind.Relative); + } + + private async Task<GhsaCursor> GetCursorAsync(CancellationToken cancellationToken) + { + var state = await _stateRepository.TryGetAsync(SourceName, cancellationToken).ConfigureAwait(false); + return state is null ? GhsaCursor.Empty : GhsaCursor.FromBson(state.Cursor); + } + + private async Task UpdateCursorAsync(GhsaCursor cursor, CancellationToken cancellationToken) + { + await _stateRepository.UpdateCursorAsync(SourceName, cursor.ToBsonDocument(), _timeProvider.GetUtcNow(), cancellationToken).ConfigureAwait(false); + } +} diff --git a/src/StellaOps.Feedser.Source.Ghsa/GhsaConnectorPlugin.cs b/src/StellaOps.Feedser.Source.Ghsa/GhsaConnectorPlugin.cs new file mode 100644 index 00000000..7f95dd9f --- /dev/null +++ b/src/StellaOps.Feedser.Source.Ghsa/GhsaConnectorPlugin.cs @@ -0,0 +1,19 @@ +using Microsoft.Extensions.DependencyInjection; +using StellaOps.Plugin; + +namespace StellaOps.Feedser.Source.Ghsa; + +public sealed class GhsaConnectorPlugin : IConnectorPlugin +{ + public const string SourceName = "ghsa"; + + public string Name => SourceName; + + public bool IsAvailable(IServiceProvider services) => services is not null; + + public IFeedConnector Create(IServiceProvider services) + { + ArgumentNullException.ThrowIfNull(services); + return ActivatorUtilities.CreateInstance<GhsaConnector>(services); + } +} diff --git a/src/StellaOps.Feedser.Source.Ghsa/GhsaDependencyInjectionRoutine.cs b/src/StellaOps.Feedser.Source.Ghsa/GhsaDependencyInjectionRoutine.cs new file mode 100644 index 00000000..86437eb5 --- /dev/null +++ b/src/StellaOps.Feedser.Source.Ghsa/GhsaDependencyInjectionRoutine.cs @@ -0,0 +1,53 @@ +using Microsoft.Extensions.Configuration; +using Microsoft.Extensions.DependencyInjection; +using StellaOps.DependencyInjection; +using StellaOps.Feedser.Core.Jobs; +using StellaOps.Feedser.Source.Ghsa.Configuration; + +namespace StellaOps.Feedser.Source.Ghsa; + +public sealed class GhsaDependencyInjectionRoutine : IDependencyInjectionRoutine +{ + private const string ConfigurationSection = "feedser:sources:ghsa"; + + public IServiceCollection Register(IServiceCollection services, IConfiguration configuration) + { + ArgumentNullException.ThrowIfNull(services); + ArgumentNullException.ThrowIfNull(configuration); + + services.AddGhsaConnector(options => + { + configuration.GetSection(ConfigurationSection).Bind(options); + options.Validate(); + }); + + services.AddTransient<GhsaFetchJob>(); + services.AddTransient<GhsaParseJob>(); + services.AddTransient<GhsaMapJob>(); + + services.PostConfigure<JobSchedulerOptions>(options => + { + EnsureJob(options, GhsaJobKinds.Fetch, typeof(GhsaFetchJob)); + EnsureJob(options, GhsaJobKinds.Parse, typeof(GhsaParseJob)); + EnsureJob(options, GhsaJobKinds.Map, typeof(GhsaMapJob)); + }); + + return services; + } + + private static void EnsureJob(JobSchedulerOptions options, string kind, Type jobType) + { + if (options.Definitions.ContainsKey(kind)) + { + return; + } + + options.Definitions[kind] = new JobDefinition( + kind, + jobType, + options.DefaultTimeout, + options.DefaultLeaseDuration, + CronExpression: null, + Enabled: true); + } +} diff --git a/src/StellaOps.Feedser.Source.Ghsa/GhsaServiceCollectionExtensions.cs b/src/StellaOps.Feedser.Source.Ghsa/GhsaServiceCollectionExtensions.cs new file mode 100644 index 00000000..7777b355 --- /dev/null +++ b/src/StellaOps.Feedser.Source.Ghsa/GhsaServiceCollectionExtensions.cs @@ -0,0 +1,37 @@ +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Options; +using StellaOps.Feedser.Source.Common.Http; +using StellaOps.Feedser.Source.Ghsa.Configuration; +using StellaOps.Feedser.Source.Ghsa.Internal; + +namespace StellaOps.Feedser.Source.Ghsa; + +public static class GhsaServiceCollectionExtensions +{ + public static IServiceCollection AddGhsaConnector(this IServiceCollection services, Action<GhsaOptions> configure) + { + ArgumentNullException.ThrowIfNull(services); + ArgumentNullException.ThrowIfNull(configure); + + services.AddOptions<GhsaOptions>() + .Configure(configure) + .PostConfigure(static opts => opts.Validate()); + + services.AddSourceHttpClient(GhsaOptions.HttpClientName, (sp, clientOptions) => + { + var options = sp.GetRequiredService<IOptions<GhsaOptions>>().Value; + clientOptions.BaseAddress = options.BaseEndpoint; + clientOptions.Timeout = TimeSpan.FromSeconds(30); + clientOptions.UserAgent = "StellaOps.Feedser.Ghsa/1.0"; + clientOptions.AllowedHosts.Clear(); + clientOptions.AllowedHosts.Add(options.BaseEndpoint.Host); + clientOptions.DefaultRequestHeaders["Accept"] = "application/vnd.github+json"; + clientOptions.DefaultRequestHeaders["Authorization"] = $"Bearer {options.ApiToken}"; + clientOptions.DefaultRequestHeaders["X-GitHub-Api-Version"] = "2022-11-28"; + }); + + services.AddSingleton<GhsaDiagnostics>(); + services.AddTransient<GhsaConnector>(); + return services; + } +} diff --git a/src/StellaOps.Feedser.Source.Ghsa/Internal/GhsaCursor.cs b/src/StellaOps.Feedser.Source.Ghsa/Internal/GhsaCursor.cs new file mode 100644 index 00000000..09f3069a --- /dev/null +++ b/src/StellaOps.Feedser.Source.Ghsa/Internal/GhsaCursor.cs @@ -0,0 +1,135 @@ +using System.Collections.Generic; +using System.Linq; +using MongoDB.Bson; + +namespace StellaOps.Feedser.Source.Ghsa.Internal; + +internal sealed record GhsaCursor( + DateTimeOffset? LastUpdatedExclusive, + DateTimeOffset? CurrentWindowStart, + DateTimeOffset? CurrentWindowEnd, + int NextPage, + IReadOnlyCollection<Guid> PendingDocuments, + IReadOnlyCollection<Guid> PendingMappings) +{ + private static readonly IReadOnlyCollection<Guid> EmptyGuidList = Array.Empty<Guid>(); + + public static GhsaCursor Empty { get; } = new( + null, + null, + null, + 1, + EmptyGuidList, + EmptyGuidList); + + public BsonDocument ToBsonDocument() + { + var document = new BsonDocument + { + ["nextPage"] = NextPage, + ["pendingDocuments"] = new BsonArray(PendingDocuments.Select(id => id.ToString())), + ["pendingMappings"] = new BsonArray(PendingMappings.Select(id => id.ToString())), + }; + + if (LastUpdatedExclusive.HasValue) + { + document["lastUpdatedExclusive"] = LastUpdatedExclusive.Value.UtcDateTime; + } + + if (CurrentWindowStart.HasValue) + { + document["currentWindowStart"] = CurrentWindowStart.Value.UtcDateTime; + } + + if (CurrentWindowEnd.HasValue) + { + document["currentWindowEnd"] = CurrentWindowEnd.Value.UtcDateTime; + } + + return document; + } + + public static GhsaCursor FromBson(BsonDocument? document) + { + if (document is null || document.ElementCount == 0) + { + return Empty; + } + + var lastUpdatedExclusive = document.TryGetValue("lastUpdatedExclusive", out var lastUpdated) + ? ParseDate(lastUpdated) + : null; + var windowStart = document.TryGetValue("currentWindowStart", out var windowStartValue) + ? ParseDate(windowStartValue) + : null; + var windowEnd = document.TryGetValue("currentWindowEnd", out var windowEndValue) + ? ParseDate(windowEndValue) + : null; + var nextPage = document.TryGetValue("nextPage", out var nextPageValue) && nextPageValue.IsInt32 + ? Math.Max(1, nextPageValue.AsInt32) + : 1; + + var pendingDocuments = ReadGuidArray(document, "pendingDocuments"); + var pendingMappings = ReadGuidArray(document, "pendingMappings"); + + return new GhsaCursor( + lastUpdatedExclusive, + windowStart, + windowEnd, + nextPage, + pendingDocuments, + pendingMappings); + } + + public GhsaCursor WithPendingDocuments(IEnumerable<Guid> ids) + => this with { PendingDocuments = ids?.Distinct().ToArray() ?? EmptyGuidList }; + + public GhsaCursor WithPendingMappings(IEnumerable<Guid> ids) + => this with { PendingMappings = ids?.Distinct().ToArray() ?? EmptyGuidList }; + + public GhsaCursor WithLastUpdatedExclusive(DateTimeOffset? timestamp) + => this with { LastUpdatedExclusive = timestamp }; + + public GhsaCursor WithCurrentWindowStart(DateTimeOffset? timestamp) + => this with { CurrentWindowStart = timestamp }; + + public GhsaCursor WithCurrentWindowEnd(DateTimeOffset? timestamp) + => this with { CurrentWindowEnd = timestamp }; + + public GhsaCursor WithNextPage(int page) + => this with { NextPage = page < 1 ? 1 : page }; + + private static DateTimeOffset? ParseDate(BsonValue value) + { + return value.BsonType switch + { + BsonType.DateTime => DateTime.SpecifyKind(value.ToUniversalTime(), DateTimeKind.Utc), + BsonType.String when DateTimeOffset.TryParse(value.AsString, out var parsed) => parsed.ToUniversalTime(), + _ => null, + }; + } + + private static IReadOnlyCollection<Guid> ReadGuidArray(BsonDocument document, string field) + { + if (!document.TryGetValue(field, out var value) || value is not BsonArray array) + { + return EmptyGuidList; + } + + var results = new List<Guid>(array.Count); + foreach (var element in array) + { + if (element is null) + { + continue; + } + + if (Guid.TryParse(element.ToString(), out var guid)) + { + results.Add(guid); + } + } + + return results; + } +} diff --git a/src/StellaOps.Feedser.Source.Ghsa/Internal/GhsaDiagnostics.cs b/src/StellaOps.Feedser.Source.Ghsa/Internal/GhsaDiagnostics.cs new file mode 100644 index 00000000..52bbabdc --- /dev/null +++ b/src/StellaOps.Feedser.Source.Ghsa/Internal/GhsaDiagnostics.cs @@ -0,0 +1,50 @@ +using System.Diagnostics.Metrics; + +namespace StellaOps.Feedser.Source.Ghsa.Internal; + +public sealed class GhsaDiagnostics : IDisposable +{ + private const string MeterName = "StellaOps.Feedser.Source.Ghsa"; + private const string MeterVersion = "1.0.0"; + + private readonly Meter _meter; + private readonly Counter<long> _fetchAttempts; + private readonly Counter<long> _fetchDocuments; + private readonly Counter<long> _fetchFailures; + private readonly Counter<long> _fetchUnchanged; + private readonly Counter<long> _parseSuccess; + private readonly Counter<long> _parseFailures; + private readonly Counter<long> _parseQuarantine; + private readonly Counter<long> _mapSuccess; + + public GhsaDiagnostics() + { + _meter = new Meter(MeterName, MeterVersion); + _fetchAttempts = _meter.CreateCounter<long>("ghsa.fetch.attempts", unit: "operations"); + _fetchDocuments = _meter.CreateCounter<long>("ghsa.fetch.documents", unit: "documents"); + _fetchFailures = _meter.CreateCounter<long>("ghsa.fetch.failures", unit: "operations"); + _fetchUnchanged = _meter.CreateCounter<long>("ghsa.fetch.unchanged", unit: "operations"); + _parseSuccess = _meter.CreateCounter<long>("ghsa.parse.success", unit: "documents"); + _parseFailures = _meter.CreateCounter<long>("ghsa.parse.failures", unit: "documents"); + _parseQuarantine = _meter.CreateCounter<long>("ghsa.parse.quarantine", unit: "documents"); + _mapSuccess = _meter.CreateCounter<long>("ghsa.map.success", unit: "advisories"); + } + + public void FetchAttempt() => _fetchAttempts.Add(1); + + public void FetchDocument() => _fetchDocuments.Add(1); + + public void FetchFailure() => _fetchFailures.Add(1); + + public void FetchUnchanged() => _fetchUnchanged.Add(1); + + public void ParseSuccess() => _parseSuccess.Add(1); + + public void ParseFailure() => _parseFailures.Add(1); + + public void ParseQuarantine() => _parseQuarantine.Add(1); + + public void MapSuccess(long count) => _mapSuccess.Add(count); + + public void Dispose() => _meter.Dispose(); +} diff --git a/src/StellaOps.Feedser.Source.Ghsa/Internal/GhsaListParser.cs b/src/StellaOps.Feedser.Source.Ghsa/Internal/GhsaListParser.cs new file mode 100644 index 00000000..034e9b5e --- /dev/null +++ b/src/StellaOps.Feedser.Source.Ghsa/Internal/GhsaListParser.cs @@ -0,0 +1,115 @@ +using System.Collections.Generic; +using System.Globalization; +using System.Text.Json; + +namespace StellaOps.Feedser.Source.Ghsa.Internal; + +internal static class GhsaListParser +{ + public static GhsaListPage Parse(ReadOnlySpan<byte> content, int currentPage, int pageSize) + { + using var document = JsonDocument.Parse(content.ToArray()); + var root = document.RootElement; + + var items = new List<GhsaListItem>(); + DateTimeOffset? maxUpdated = null; + + if (root.TryGetProperty("advisories", out var advisories) && advisories.ValueKind == JsonValueKind.Array) + { + foreach (var advisory in advisories.EnumerateArray()) + { + if (advisory.ValueKind != JsonValueKind.Object) + { + continue; + } + + var id = GetString(advisory, "ghsa_id"); + if (string.IsNullOrWhiteSpace(id)) + { + continue; + } + + var updated = GetDate(advisory, "updated_at"); + if (updated.HasValue && (!maxUpdated.HasValue || updated > maxUpdated)) + { + maxUpdated = updated; + } + + items.Add(new GhsaListItem(id, updated)); + } + } + + var hasMorePages = TryDetermineHasMore(root, currentPage, pageSize, items.Count, out var nextPage); + + return new GhsaListPage(items, maxUpdated, hasMorePages, nextPage ?? currentPage + 1); + } + + private static bool TryDetermineHasMore(JsonElement root, int currentPage, int pageSize, int itemCount, out int? nextPage) + { + nextPage = null; + + if (root.TryGetProperty("pagination", out var pagination) && pagination.ValueKind == JsonValueKind.Object) + { + var hasNextPage = pagination.TryGetProperty("has_next_page", out var hasNext) && hasNext.ValueKind == JsonValueKind.True; + if (hasNextPage) + { + nextPage = currentPage + 1; + return true; + } + + if (pagination.TryGetProperty("total_pages", out var totalPagesElement) && totalPagesElement.ValueKind == JsonValueKind.Number && totalPagesElement.TryGetInt32(out var totalPages)) + { + if (currentPage < totalPages) + { + nextPage = currentPage + 1; + return true; + } + } + + return false; + } + + if (itemCount >= pageSize) + { + nextPage = currentPage + 1; + return true; + } + + return false; + } + + private static string? GetString(JsonElement element, string propertyName) + { + if (!element.TryGetProperty(propertyName, out var property)) + { + return null; + } + + return property.ValueKind switch + { + JsonValueKind.String => property.GetString(), + _ => null, + }; + } + + private static DateTimeOffset? GetDate(JsonElement element, string propertyName) + { + var value = GetString(element, propertyName); + if (string.IsNullOrWhiteSpace(value)) + { + return null; + } + + return DateTimeOffset.TryParse(value, CultureInfo.InvariantCulture, DateTimeStyles.AssumeUniversal, out var parsed) + ? parsed.ToUniversalTime() + : null; + } +} + +internal sealed record GhsaListPage( + IReadOnlyList<GhsaListItem> Items, + DateTimeOffset? MaxUpdated, + bool HasMorePages, + int NextPageCandidate); + +internal sealed record GhsaListItem(string GhsaId, DateTimeOffset? UpdatedAt); diff --git a/src/StellaOps.Feedser.Source.Ghsa/Internal/GhsaMapper.cs b/src/StellaOps.Feedser.Source.Ghsa/Internal/GhsaMapper.cs new file mode 100644 index 00000000..a0e2acd5 --- /dev/null +++ b/src/StellaOps.Feedser.Source.Ghsa/Internal/GhsaMapper.cs @@ -0,0 +1,209 @@ +using System.Collections.Generic; +using System.Linq; +using StellaOps.Feedser.Models; +using StellaOps.Feedser.Storage.Mongo.Documents; + +namespace StellaOps.Feedser.Source.Ghsa.Internal; + +internal static class GhsaMapper +{ + private static readonly HashSet<string> SemVerEcosystems = new(StringComparer.OrdinalIgnoreCase) + { + "npm", + "maven", + "pip", + "rubygems", + "composer", + "nuget", + "go", + "cargo", + }; + + public static Advisory Map(GhsaRecordDto dto, DocumentRecord document, DateTimeOffset recordedAt) + { + ArgumentNullException.ThrowIfNull(dto); + ArgumentNullException.ThrowIfNull(document); + + var fetchProvenance = new AdvisoryProvenance( + GhsaConnectorPlugin.SourceName, + "document", + document.Uri, + document.FetchedAt, + new[] { ProvenanceFieldMasks.Advisory }); + var mapProvenance = new AdvisoryProvenance( + GhsaConnectorPlugin.SourceName, + "mapping", + dto.GhsaId, + recordedAt, + new[] { ProvenanceFieldMasks.Advisory }); + + var aliases = dto.Aliases + .Where(static alias => !string.IsNullOrWhiteSpace(alias)) + .Distinct(StringComparer.OrdinalIgnoreCase) + .ToArray(); + + var references = dto.References + .Select(reference => CreateReference(reference, recordedAt)) + .Where(static reference => reference is not null) + .Cast<AdvisoryReference>() + .ToList(); + + var affected = CreateAffectedPackages(dto, recordedAt); + var credits = CreateCredits(dto.Credits, recordedAt); + + var severity = dto.Severity?.ToLowerInvariant(); + var summary = dto.Summary ?? dto.Description; + + return new Advisory( + advisoryKey: dto.GhsaId, + title: dto.Summary ?? dto.GhsaId, + summary: summary, + language: "en", + published: dto.PublishedAt, + modified: dto.UpdatedAt ?? dto.PublishedAt, + severity: severity, + exploitKnown: false, + aliases: aliases, + credits: credits, + references: references, + affectedPackages: affected, + cvssMetrics: Array.Empty<CvssMetric>(), + provenance: new[] { fetchProvenance, mapProvenance }); + } + + private static AdvisoryReference? CreateReference(GhsaReferenceDto reference, DateTimeOffset recordedAt) + { + if (string.IsNullOrWhiteSpace(reference.Url) || !Validation.LooksLikeHttpUrl(reference.Url)) + { + return null; + } + + var kind = reference.Type?.ToLowerInvariant(); + + return new AdvisoryReference( + reference.Url, + kind, + reference.Name, + summary: null, + provenance: new AdvisoryProvenance( + GhsaConnectorPlugin.SourceName, + "reference", + reference.Url, + recordedAt, + new[] { ProvenanceFieldMasks.References })); + } + + private static IReadOnlyList<AffectedPackage> CreateAffectedPackages(GhsaRecordDto dto, DateTimeOffset recordedAt) + { + if (dto.Affected.Count == 0) + { + return Array.Empty<AffectedPackage>(); + } + + var packages = new List<AffectedPackage>(dto.Affected.Count); + foreach (var affected in dto.Affected) + { + var ecosystem = string.IsNullOrWhiteSpace(affected.Ecosystem) ? "unknown" : affected.Ecosystem.Trim(); + var packageName = string.IsNullOrWhiteSpace(affected.PackageName) ? "unknown-package" : affected.PackageName.Trim(); + var identifier = $"{ecosystem.ToLowerInvariant()}:{packageName}"; + + var provenance = new[] + { + new AdvisoryProvenance( + GhsaConnectorPlugin.SourceName, + "affected", + identifier, + recordedAt, + new[] { ProvenanceFieldMasks.AffectedPackages }), + }; + + var rangeKind = SemVerEcosystems.Contains(ecosystem) ? "semver" : "vendor"; + var packageType = SemVerEcosystems.Contains(ecosystem) ? AffectedPackageTypes.SemVer : AffectedPackageTypes.Vendor; + + var versionRanges = new List<AffectedVersionRange>(); + if (!string.IsNullOrWhiteSpace(affected.VulnerableRange) || !string.IsNullOrWhiteSpace(affected.PatchedVersion)) + { + var primitives = new RangePrimitives(null, null, null, new Dictionary<string, string>(StringComparer.OrdinalIgnoreCase) + { + ["ecosystem"] = ecosystem, + ["package"] = packageName, + }); + + versionRanges.Add(new AffectedVersionRange( + rangeKind, + introducedVersion: null, + fixedVersion: Validation.TrimToNull(affected.PatchedVersion), + lastAffectedVersion: null, + rangeExpression: Validation.TrimToNull(affected.VulnerableRange), + provenance: new AdvisoryProvenance( + GhsaConnectorPlugin.SourceName, + "affected-range", + identifier, + recordedAt, + new[] { ProvenanceFieldMasks.VersionRanges }), + primitives: primitives)); + } + + var statuses = new[] + { + new AffectedPackageStatus( + "affected", + new AdvisoryProvenance( + GhsaConnectorPlugin.SourceName, + "affected-status", + identifier, + recordedAt, + new[] { ProvenanceFieldMasks.PackageStatuses })), + }; + + packages.Add(new AffectedPackage( + packageType, + identifier, + platform: null, + versionRanges: versionRanges, + statuses: statuses, + provenance: provenance)); + } + + return packages; + } + + private static IReadOnlyList<AdvisoryCredit> CreateCredits(IReadOnlyList<GhsaCreditDto> credits, DateTimeOffset recordedAt) + { + if (credits.Count == 0) + { + return Array.Empty<AdvisoryCredit>(); + } + + var results = new List<AdvisoryCredit>(credits.Count); + foreach (var credit in credits) + { + var displayName = Validation.TrimToNull(credit.Name) ?? Validation.TrimToNull(credit.Login); + if (displayName is null) + { + continue; + } + + var contacts = new List<string>(); + if (!string.IsNullOrWhiteSpace(credit.ProfileUrl) && Validation.LooksLikeHttpUrl(credit.ProfileUrl)) + { + contacts.Add(credit.ProfileUrl.Trim()); + } + else if (!string.IsNullOrWhiteSpace(credit.Login)) + { + contacts.Add($"https://github.com/{credit.Login.Trim()}"); + } + + var provenance = new AdvisoryProvenance( + GhsaConnectorPlugin.SourceName, + "credit", + displayName, + recordedAt, + new[] { ProvenanceFieldMasks.Credits }); + + results.Add(new AdvisoryCredit(displayName, credit.Type, contacts, provenance)); + } + + return results.Count == 0 ? Array.Empty<AdvisoryCredit>() : results; + } +} diff --git a/src/StellaOps.Feedser.Source.Ghsa/Internal/GhsaRecordDto.cs b/src/StellaOps.Feedser.Source.Ghsa/Internal/GhsaRecordDto.cs new file mode 100644 index 00000000..1f5e6653 --- /dev/null +++ b/src/StellaOps.Feedser.Source.Ghsa/Internal/GhsaRecordDto.cs @@ -0,0 +1,55 @@ +namespace StellaOps.Feedser.Source.Ghsa.Internal; + +internal sealed record GhsaRecordDto +{ + public string GhsaId { get; init; } = string.Empty; + + public string? Summary { get; init; } + + public string? Description { get; init; } + + public string? Severity { get; init; } + + public DateTimeOffset? PublishedAt { get; init; } + + public DateTimeOffset? UpdatedAt { get; init; } + + public IReadOnlyList<string> Aliases { get; init; } = Array.Empty<string>(); + + public IReadOnlyList<GhsaReferenceDto> References { get; init; } = Array.Empty<GhsaReferenceDto>(); + + public IReadOnlyList<GhsaAffectedDto> Affected { get; init; } = Array.Empty<GhsaAffectedDto>(); + + public IReadOnlyList<GhsaCreditDto> Credits { get; init; } = Array.Empty<GhsaCreditDto>(); +} + +internal sealed record GhsaReferenceDto +{ + public string Url { get; init; } = string.Empty; + + public string? Type { get; init; } + + public string? Name { get; init; } +} + +internal sealed record GhsaAffectedDto +{ + public string PackageName { get; init; } = string.Empty; + + public string Ecosystem { get; init; } = string.Empty; + + public string? VulnerableRange { get; init; } + + public string? PatchedVersion { get; init; } +} + +internal sealed record GhsaCreditDto +{ + public string? Type { get; init; } + + public string? Name { get; init; } + + public string? Login { get; init; } + + public string? ProfileUrl { get; init; } +} diff --git a/src/StellaOps.Feedser.Source.Ghsa/Internal/GhsaRecordParser.cs b/src/StellaOps.Feedser.Source.Ghsa/Internal/GhsaRecordParser.cs new file mode 100644 index 00000000..5c05db1a --- /dev/null +++ b/src/StellaOps.Feedser.Source.Ghsa/Internal/GhsaRecordParser.cs @@ -0,0 +1,205 @@ +using System.Collections.Generic; +using System.Globalization; +using System.Text.Json; + +namespace StellaOps.Feedser.Source.Ghsa.Internal; + +internal static class GhsaRecordParser +{ + public static GhsaRecordDto Parse(ReadOnlySpan<byte> content) + { + using var document = JsonDocument.Parse(content.ToArray()); + var root = document.RootElement; + + var ghsaId = GetString(root, "ghsa_id") ?? throw new JsonException("ghsa_id missing"); + var summary = GetString(root, "summary"); + var description = GetString(root, "description"); + var severity = GetString(root, "severity"); + var publishedAt = GetDate(root, "published_at"); + var updatedAt = GetDate(root, "updated_at") ?? publishedAt; + + var aliases = new HashSet<string>(StringComparer.OrdinalIgnoreCase) + { + ghsaId, + }; + + if (root.TryGetProperty("cve_ids", out var cveIds) && cveIds.ValueKind == JsonValueKind.Array) + { + foreach (var cve in cveIds.EnumerateArray()) + { + if (cve.ValueKind == JsonValueKind.String && !string.IsNullOrWhiteSpace(cve.GetString())) + { + aliases.Add(cve.GetString()!); + } + } + } + + var references = ParseReferences(root); + var affected = ParseAffected(root); + var credits = ParseCredits(root); + + return new GhsaRecordDto + { + GhsaId = ghsaId, + Summary = summary, + Description = description, + Severity = severity, + PublishedAt = publishedAt, + UpdatedAt = updatedAt, + Aliases = aliases.ToArray(), + References = references, + Affected = affected, + Credits = credits, + }; + } + + private static IReadOnlyList<GhsaReferenceDto> ParseReferences(JsonElement root) + { + if (!root.TryGetProperty("references", out var references) || references.ValueKind != JsonValueKind.Array) + { + return Array.Empty<GhsaReferenceDto>(); + } + + var list = new List<GhsaReferenceDto>(references.GetArrayLength()); + foreach (var reference in references.EnumerateArray()) + { + if (reference.ValueKind != JsonValueKind.Object) + { + continue; + } + + var url = GetString(reference, "url"); + if (string.IsNullOrWhiteSpace(url)) + { + continue; + } + + list.Add(new GhsaReferenceDto + { + Url = url, + Type = GetString(reference, "type"), + Name = GetString(reference, "name"), + }); + } + + return list; + } + + private static IReadOnlyList<GhsaAffectedDto> ParseAffected(JsonElement root) + { + if (!root.TryGetProperty("vulnerabilities", out var vulnerabilities) || vulnerabilities.ValueKind != JsonValueKind.Array) + { + return Array.Empty<GhsaAffectedDto>(); + } + + var list = new List<GhsaAffectedDto>(vulnerabilities.GetArrayLength()); + foreach (var entry in vulnerabilities.EnumerateArray()) + { + if (entry.ValueKind != JsonValueKind.Object) + { + continue; + } + + var package = entry.TryGetProperty("package", out var packageElement) && packageElement.ValueKind == JsonValueKind.Object + ? packageElement + : default; + + var packageName = GetString(package, "name") ?? "unknown-package"; + var ecosystem = GetString(package, "ecosystem") ?? "unknown"; + var vulnerableRange = GetString(entry, "vulnerable_version_range"); + + string? patchedVersion = null; + if (entry.TryGetProperty("first_patched_version", out var patchedElement) && patchedElement.ValueKind == JsonValueKind.Object) + { + patchedVersion = GetString(patchedElement, "identifier"); + } + + list.Add(new GhsaAffectedDto + { + PackageName = packageName, + Ecosystem = ecosystem, + VulnerableRange = vulnerableRange, + PatchedVersion = patchedVersion, + }); + } + + return list; + } + + private static IReadOnlyList<GhsaCreditDto> ParseCredits(JsonElement root) + { + if (!root.TryGetProperty("credits", out var credits) || credits.ValueKind != JsonValueKind.Array) + { + return Array.Empty<GhsaCreditDto>(); + } + + var list = new List<GhsaCreditDto>(credits.GetArrayLength()); + foreach (var credit in credits.EnumerateArray()) + { + if (credit.ValueKind != JsonValueKind.Object) + { + continue; + } + + var type = GetString(credit, "type"); + var name = GetString(credit, "name"); + string? login = null; + string? profile = null; + + if (credit.TryGetProperty("user", out var user) && user.ValueKind == JsonValueKind.Object) + { + login = GetString(user, "login"); + profile = GetString(user, "html_url") ?? GetString(user, "url"); + name ??= GetString(user, "name"); + } + + name ??= login; + if (string.IsNullOrWhiteSpace(name)) + { + continue; + } + + list.Add(new GhsaCreditDto + { + Type = type, + Name = name, + Login = login, + ProfileUrl = profile, + }); + } + + return list; + } + + private static string? GetString(JsonElement element, string propertyName) + { + if (element.ValueKind != JsonValueKind.Object) + { + return null; + } + + if (!element.TryGetProperty(propertyName, out var property)) + { + return null; + } + + return property.ValueKind switch + { + JsonValueKind.String => property.GetString(), + _ => null, + }; + } + + private static DateTimeOffset? GetDate(JsonElement element, string propertyName) + { + var value = GetString(element, propertyName); + if (string.IsNullOrWhiteSpace(value)) + { + return null; + } + + return DateTimeOffset.TryParse(value, CultureInfo.InvariantCulture, DateTimeStyles.AssumeUniversal, out var parsed) + ? parsed.ToUniversalTime() + : null; + } +} diff --git a/src/StellaOps.Feedser.Source.Ghsa/Jobs.cs b/src/StellaOps.Feedser.Source.Ghsa/Jobs.cs new file mode 100644 index 00000000..67759407 --- /dev/null +++ b/src/StellaOps.Feedser.Source.Ghsa/Jobs.cs @@ -0,0 +1,43 @@ +using StellaOps.Feedser.Core.Jobs; + +namespace StellaOps.Feedser.Source.Ghsa; + +internal static class GhsaJobKinds +{ + public const string Fetch = "source:ghsa:fetch"; + public const string Parse = "source:ghsa:parse"; + public const string Map = "source:ghsa:map"; +} + +internal sealed class GhsaFetchJob : IJob +{ + private readonly GhsaConnector _connector; + + public GhsaFetchJob(GhsaConnector connector) + => _connector = connector ?? throw new ArgumentNullException(nameof(connector)); + + public Task ExecuteAsync(JobExecutionContext context, CancellationToken cancellationToken) + => _connector.FetchAsync(context.Services, cancellationToken); +} + +internal sealed class GhsaParseJob : IJob +{ + private readonly GhsaConnector _connector; + + public GhsaParseJob(GhsaConnector connector) + => _connector = connector ?? throw new ArgumentNullException(nameof(connector)); + + public Task ExecuteAsync(JobExecutionContext context, CancellationToken cancellationToken) + => _connector.ParseAsync(context.Services, cancellationToken); +} + +internal sealed class GhsaMapJob : IJob +{ + private readonly GhsaConnector _connector; + + public GhsaMapJob(GhsaConnector connector) + => _connector = connector ?? throw new ArgumentNullException(nameof(connector)); + + public Task ExecuteAsync(JobExecutionContext context, CancellationToken cancellationToken) + => _connector.MapAsync(context.Services, cancellationToken); +} diff --git a/src/StellaOps.Feedser.Source.Ghsa/Properties/AssemblyInfo.cs b/src/StellaOps.Feedser.Source.Ghsa/Properties/AssemblyInfo.cs new file mode 100644 index 00000000..150bab12 --- /dev/null +++ b/src/StellaOps.Feedser.Source.Ghsa/Properties/AssemblyInfo.cs @@ -0,0 +1,3 @@ +using System.Runtime.CompilerServices; + +[assembly: InternalsVisibleTo("FixtureUpdater")] diff --git a/src/StellaOps.Feedser.Source.Ghsa/StellaOps.Feedser.Source.Ghsa.csproj b/src/StellaOps.Feedser.Source.Ghsa/StellaOps.Feedser.Source.Ghsa.csproj new file mode 100644 index 00000000..f7f2c154 --- /dev/null +++ b/src/StellaOps.Feedser.Source.Ghsa/StellaOps.Feedser.Source.Ghsa.csproj @@ -0,0 +1,16 @@ +<Project Sdk="Microsoft.NET.Sdk"> + + <PropertyGroup> + <TargetFramework>net10.0</TargetFramework> + <ImplicitUsings>enable</ImplicitUsings> + <Nullable>enable</Nullable> + </PropertyGroup> + + <ItemGroup> + <ProjectReference Include="../StellaOps.Plugin/StellaOps.Plugin.csproj" /> + + <ProjectReference Include="../StellaOps.Feedser.Source.Common/StellaOps.Feedser.Source.Common.csproj" /> + <ProjectReference Include="../StellaOps.Feedser.Models/StellaOps.Feedser.Models.csproj" /> + </ItemGroup> +</Project> + diff --git a/src/StellaOps.Feedser.Source.Ghsa/TASKS.md b/src/StellaOps.Feedser.Source.Ghsa/TASKS.md new file mode 100644 index 00000000..f71df162 --- /dev/null +++ b/src/StellaOps.Feedser.Source.Ghsa/TASKS.md @@ -0,0 +1,13 @@ +# TASKS +| Task | Owner(s) | Depends on | Notes | +|---|---|---|---| +|Select GHSA data source & auth model|BE-Conn-GHSA|Research|**DONE (2025-10-10)** – Adopted GitHub Security Advisories REST (global) endpoint with bearer token + API version headers documented in `GhsaOptions`.| +|Fetch pipeline & state management|BE-Conn-GHSA|Source.Common, Storage.Mongo|**DONE (2025-10-10)** – Implemented list/detail fetch using `GhsaCursor` (time window + page), resumable SourceState and backoff controls.| +|DTO & parser implementation|BE-Conn-GHSA|Source.Common|**DONE (2025-10-10)** – Added `GhsaRecordParser`/DTOs extracting aliases, references, severity, vulnerable ranges, patched versions.| +|Canonical mapping & range primitives|BE-Conn-GHSA|Models|**DONE (2025-10-10)** – `GhsaMapper` emits GHSA advisories with SemVer packages, vendor extensions (ecosystem/package) and deterministic references.| +|Deterministic fixtures & tests|QA|Testing|**DONE (2025-10-10)** – New `StellaOps.Feedser.Source.Ghsa.Tests` regression covers fetch/parse/map via canned GHSA fixtures and snapshot assertions.| +|Telemetry & documentation|DevEx|Docs|**DONE (2025-10-10)** – Diagnostics meter (`ghsa.fetch.*`) wired; DI extension documents token/headers and job registrations.| +|GitHub quota monitoring & retries|BE-Conn-GHSA, Observability|Source.Common|**TODO** – Add rate-limit dashboard/alerts consuming `X-RateLimit-*` headers, tune retry/backoff strategy for 403/secondary rate limits, and document mitigation steps.| +|Production credential & scheduler rollout|Ops, BE-Conn-GHSA|Docs, WebService|**TODO** – Issue PAT for production, update Helm/compose secrets, register fetch/parse/map cron defaults, and schedule staged backfill with health checks.| +|FEEDCONN-GHSA-02-004 GHSA credits & ecosystem severity mapping|BE-Conn-GHSA|Models `FEEDMODELS-SCHEMA-01-002`|**DONE (2025-10-11)** – Mapper emits advisory credits with provenance masks, fixtures assert role/contact ordering, and severity normalization remains unchanged.| +|FEEDCONN-GHSA-02-007 Credit parity regression fixtures|BE-Conn-GHSA, QA|Source.Nvd, Source.Osv|**TODO** – Capture GHSA/NVD/OSV triples with acknowledgements, wire conflict tests, and document fixture refresh workflow in `docs/dev/fixtures.md`.| diff --git a/src/StellaOps.Feedser.Source.Ics.Cisa/AGENTS.md b/src/StellaOps.Feedser.Source.Ics.Cisa/AGENTS.md new file mode 100644 index 00000000..01ca3f43 --- /dev/null +++ b/src/StellaOps.Feedser.Source.Ics.Cisa/AGENTS.md @@ -0,0 +1,39 @@ +# AGENTS +## Role +Implement the CISA ICS advisory connector to ingest US CISA Industrial Control Systems advisories (distinct from the general CERT feed). + +## Scope +- Locate the official CISA ICS advisory feed/API (currently HTML/RSS) and define fetch cadence/windowing. +- Build fetch/cursor pipeline with retry/backoff and raw document storage. +- Parse advisory content for summary, impacted vendors/products, mitigation, CVEs. +- Map advisories into canonical `Advisory` records with aliases, references, affected ICS packages, and range primitives. +- Provide deterministic fixtures and automated regression tests. + +## Participants +- `Source.Common` (HTTP/fetch utilities, DTO storage). +- `Storage.Mongo` (raw/document/DTO/advisory stores + source state). +- `Feedser.Models` (canonical advisory structures). +- `Feedser.Testing` (integration fixtures and snapshots). + +## Interfaces & Contracts +- Job kinds: `ics-cisa:fetch`, `ics-cisa:parse`, `ics-cisa:map`. +- Persist upstream caching metadata (ETag/Last-Modified) when available. +- Alias set should include CISA ICS advisory IDs and referenced CVE IDs. + +## In/Out of scope +In scope: +- ICS-specific advisories from CISA. +- Range primitives capturing vendor/equipment metadata. + +Out of scope: +- General CISA alerts (covered elsewhere). + +## Observability & Security Expectations +- Log fetch attempts, advisory counts, and mapping results. +- Sanitize HTML, removing scripts/styles before persistence. +- Honour upstream rate limits with exponential backoff. + +## Tests +- Add `StellaOps.Feedser.Source.Ics.Cisa.Tests` to cover fetch/parse/map with canned fixtures. +- Snapshot canonical advisories; support fixture regeneration via env flag. +- Ensure deterministic ordering/time normalisation. diff --git a/src/StellaOps.Feedser.Source.Ics.Cisa/Class1.cs b/src/StellaOps.Feedser.Source.Ics.Cisa/Class1.cs new file mode 100644 index 00000000..208f522d --- /dev/null +++ b/src/StellaOps.Feedser.Source.Ics.Cisa/Class1.cs @@ -0,0 +1,29 @@ +using System; +using System.Threading; +using System.Threading.Tasks; +using StellaOps.Plugin; + +namespace StellaOps.Feedser.Source.Ics.Cisa; + +public sealed class IcsCisaConnectorPlugin : IConnectorPlugin +{ + public string Name => "ics-cisa"; + + public bool IsAvailable(IServiceProvider services) => true; + + public IFeedConnector Create(IServiceProvider services) => new StubConnector(Name); + + private sealed class StubConnector : IFeedConnector + { + public StubConnector(string sourceName) => SourceName = sourceName; + + public string SourceName { get; } + + public Task FetchAsync(IServiceProvider services, CancellationToken cancellationToken) => Task.CompletedTask; + + public Task ParseAsync(IServiceProvider services, CancellationToken cancellationToken) => Task.CompletedTask; + + public Task MapAsync(IServiceProvider services, CancellationToken cancellationToken) => Task.CompletedTask; + } +} + diff --git a/src/StellaOps.Feedser.Source.Ics.Cisa/StellaOps.Feedser.Source.Ics.Cisa.csproj b/src/StellaOps.Feedser.Source.Ics.Cisa/StellaOps.Feedser.Source.Ics.Cisa.csproj new file mode 100644 index 00000000..f7f2c154 --- /dev/null +++ b/src/StellaOps.Feedser.Source.Ics.Cisa/StellaOps.Feedser.Source.Ics.Cisa.csproj @@ -0,0 +1,16 @@ +<Project Sdk="Microsoft.NET.Sdk"> + + <PropertyGroup> + <TargetFramework>net10.0</TargetFramework> + <ImplicitUsings>enable</ImplicitUsings> + <Nullable>enable</Nullable> + </PropertyGroup> + + <ItemGroup> + <ProjectReference Include="../StellaOps.Plugin/StellaOps.Plugin.csproj" /> + + <ProjectReference Include="../StellaOps.Feedser.Source.Common/StellaOps.Feedser.Source.Common.csproj" /> + <ProjectReference Include="../StellaOps.Feedser.Models/StellaOps.Feedser.Models.csproj" /> + </ItemGroup> +</Project> + diff --git a/src/StellaOps.Feedser.Source.Ics.Cisa/TASKS.md b/src/StellaOps.Feedser.Source.Ics.Cisa/TASKS.md new file mode 100644 index 00000000..305d0857 --- /dev/null +++ b/src/StellaOps.Feedser.Source.Ics.Cisa/TASKS.md @@ -0,0 +1,11 @@ +# TASKS +| Task | Owner(s) | Depends on | Notes | +|---|---|---|---| +|FEEDCONN-ICSCISA-02-001 Document CISA ICS feed contract|BE-Conn-ICS-CISA|Research|**DOING** – 2025-10-11: Verified new ICS advisory RSS endpoint (`https://www.cisa.gov/cybersecurity-advisories/ics-advisories.xml`) and logged Akamai “Access Denied” responses; need cadence/retention detail, official statement on RSS availability, and contingency if feed requires GovDelivery tokens.| +|FEEDCONN-ICSCISA-02-002 Fetch pipeline & cursor storage|BE-Conn-ICS-CISA|Source.Common, Storage.Mongo|**TODO** – Implement HTTP client, fetch job, dedupe, and state persistence with backoff handling.| +|FEEDCONN-ICSCISA-02-003 DTO/parser implementation|BE-Conn-ICS-CISA|Source.Common|**TODO** – Create DTOs parsing ICS advisories, extract vendors, products, mitigation steps, references, CVEs.| +|FEEDCONN-ICSCISA-02-004 Canonical mapping & range primitives|BE-Conn-ICS-CISA|Models|**TODO** – Map advisories into canonical records with aliases, references, affected ICS vendor packages, and range primitives. Reference normalized rule expectations in `../StellaOps.Feedser.Merge/RANGE_PRIMITIVES_COORDINATION.md`.| +|FEEDCONN-ICSCISA-02-005 Deterministic fixtures/tests|QA|Testing|**TODO** – Provide fetch/parse/map regression tests; support `UPDATE_ICS_CISA_FIXTURES=1`.| +|FEEDCONN-ICSCISA-02-006 Telemetry & documentation|DevEx|Docs|**TODO** – Add logging/metrics, update module documentation, and close backlog when production ready.| +|FEEDCONN-ICSCISA-02-007 Detail document inventory|BE-Conn-ICS-CISA|Research|**TODO** – Catalogue HTML/PDF detail URLs per advisory and specify sanitization/attachment strategy for DTO stage.| +|FEEDCONN-ICSCISA-02-008 Distribution fallback strategy|BE-Conn-ICS-CISA|Research|**TODO** – Confirm CISA’s post-2025 RSS availability with GovDelivery program; outline alternative ingestion (email, HTML scrape, partner API) if RSS remains access-restricted.| diff --git a/src/StellaOps.Feedser.Source.Ics.Kaspersky.Tests/Kaspersky/Fixtures/detail-acme-controller-2024.html b/src/StellaOps.Feedser.Source.Ics.Kaspersky.Tests/Kaspersky/Fixtures/detail-acme-controller-2024.html new file mode 100644 index 00000000..76572e0a --- /dev/null +++ b/src/StellaOps.Feedser.Source.Ics.Kaspersky.Tests/Kaspersky/Fixtures/detail-acme-controller-2024.html @@ -0,0 +1,18 @@ +<!DOCTYPE html> +<html lang="en"> + <head> + <meta charset="utf-8" /> + <title>ACME Corp controllers multiple vulnerabilities + + +
    +

    ACME Corp controllers multiple vulnerabilities

    +

    Researchers identified that ACME Corp ICS controller models X100 and X200 are affected by improper access controls.

    +

    Exploitation of CVE-2024-7777 can allow authenticated attackers to execute arbitrary commands. Additional details are provided in CVE-2024-8888.

    +
      +
    • Vendor: ACME Corp
      • +
    • Affected models: X100, X200
      • +
    +
    + + diff --git a/src/StellaOps.Feedser.Source.Ics.Kaspersky.Tests/Kaspersky/Fixtures/expected-advisory.json b/src/StellaOps.Feedser.Source.Ics.Kaspersky.Tests/Kaspersky/Fixtures/expected-advisory.json new file mode 100644 index 00000000..b2985542 --- /dev/null +++ b/src/StellaOps.Feedser.Source.Ics.Kaspersky.Tests/Kaspersky/Fixtures/expected-advisory.json @@ -0,0 +1,515 @@ +{ + "advisoryKey": "acme-controller-2024", + "affectedPackages": [ + { + "identifier": "2024", + "platform": null, + "provenance": [ + { + "fieldMask": [], + "kind": "affected", + "recordedAt": "2024-10-20T00:01:00+00:00", + "source": "ics-kaspersky", + "value": "2024" + } + ], + "statuses": [], + "type": "ics-vendor", + "versionRanges": [ + { + "fixedVersion": null, + "introducedVersion": null, + "lastAffectedVersion": null, + "primitives": { + "evr": null, + "hasVendorExtensions": true, + "nevra": null, + "semVer": null, + "vendorExtensions": { + "ics.vendor": "2024" + } + }, + "provenance": { + "fieldMask": [], + "kind": "affected", + "recordedAt": "2024-10-20T00:01:00+00:00", + "source": "ics-kaspersky", + "value": "2024" + }, + "rangeExpression": null, + "rangeKind": "vendor" + } + ] + }, + { + "identifier": "7777 can allow authenticated attackers to execute arbitrary commands", + "platform": null, + "provenance": [ + { + "fieldMask": [], + "kind": "affected", + "recordedAt": "2024-10-20T00:01:00+00:00", + "source": "ics-kaspersky", + "value": "7777 can allow authenticated attackers to execute arbitrary commands" + } + ], + "statuses": [], + "type": "ics-vendor", + "versionRanges": [ + { + "fixedVersion": null, + "introducedVersion": null, + "lastAffectedVersion": null, + "primitives": { + "evr": null, + "hasVendorExtensions": true, + "nevra": null, + "semVer": null, + "vendorExtensions": { + "ics.vendor": "7777 can allow authenticated attackers to execute arbitrary commands" + } + }, + "provenance": { + "fieldMask": [], + "kind": "affected", + "recordedAt": "2024-10-20T00:01:00+00:00", + "source": "ics-kaspersky", + "value": "7777 can allow authenticated attackers to execute arbitrary commands" + }, + "rangeExpression": null, + "rangeKind": "vendor" + } + ] + }, + { + "identifier": "7777)", + "platform": null, + "provenance": [ + { + "fieldMask": [], + "kind": "affected", + "recordedAt": "2024-10-20T00:01:00+00:00", + "source": "ics-kaspersky", + "value": "7777)" + } + ], + "statuses": [], + "type": "ics-vendor", + "versionRanges": [ + { + "fixedVersion": null, + "introducedVersion": null, + "lastAffectedVersion": null, + "primitives": { + "evr": null, + "hasVendorExtensions": true, + "nevra": null, + "semVer": null, + "vendorExtensions": { + "ics.vendor": "7777)" + } + }, + "provenance": { + "fieldMask": [], + "kind": "affected", + "recordedAt": "2024-10-20T00:01:00+00:00", + "source": "ics-kaspersky", + "value": "7777)" + }, + "rangeExpression": null, + "rangeKind": "vendor" + } + ] + }, + { + "identifier": "8888", + "platform": null, + "provenance": [ + { + "fieldMask": [], + "kind": "affected", + "recordedAt": "2024-10-20T00:01:00+00:00", + "source": "ics-kaspersky", + "value": "8888" + } + ], + "statuses": [], + "type": "ics-vendor", + "versionRanges": [ + { + "fixedVersion": null, + "introducedVersion": null, + "lastAffectedVersion": null, + "primitives": { + "evr": null, + "hasVendorExtensions": true, + "nevra": null, + "semVer": null, + "vendorExtensions": { + "ics.vendor": "8888" + } + }, + "provenance": { + "fieldMask": [], + "kind": "affected", + "recordedAt": "2024-10-20T00:01:00+00:00", + "source": "ics-kaspersky", + "value": "8888" + }, + "rangeExpression": null, + "rangeKind": "vendor" + } + ] + }, + { + "identifier": "ACME Corp", + "platform": null, + "provenance": [ + { + "fieldMask": [], + "kind": "affected", + "recordedAt": "2024-10-20T00:01:00+00:00", + "source": "ics-kaspersky", + "value": "ACME Corp" + } + ], + "statuses": [], + "type": "ics-vendor", + "versionRanges": [ + { + "fixedVersion": null, + "introducedVersion": null, + "lastAffectedVersion": null, + "primitives": { + "evr": null, + "hasVendorExtensions": true, + "nevra": null, + "semVer": null, + "vendorExtensions": { + "ics.vendor": "ACME Corp" + } + }, + "provenance": { + "fieldMask": [], + "kind": "affected", + "recordedAt": "2024-10-20T00:01:00+00:00", + "source": "ics-kaspersky", + "value": "ACME Corp" + }, + "rangeExpression": null, + "rangeKind": "vendor" + } + ] + }, + { + "identifier": "ACME Corp Affected models", + "platform": null, + "provenance": [ + { + "fieldMask": [], + "kind": "affected", + "recordedAt": "2024-10-20T00:01:00+00:00", + "source": "ics-kaspersky", + "value": "ACME Corp Affected models" + } + ], + "statuses": [], + "type": "ics-vendor", + "versionRanges": [ + { + "fixedVersion": null, + "introducedVersion": null, + "lastAffectedVersion": null, + "primitives": { + "evr": null, + "hasVendorExtensions": true, + "nevra": null, + "semVer": null, + "vendorExtensions": { + "ics.vendor": "ACME Corp Affected models" + } + }, + "provenance": { + "fieldMask": [], + "kind": "affected", + "recordedAt": "2024-10-20T00:01:00+00:00", + "source": "ics-kaspersky", + "value": "ACME Corp Affected models" + }, + "rangeExpression": null, + "rangeKind": "vendor" + } + ] + }, + { + "identifier": "ACME Corp industrial", + "platform": null, + "provenance": [ + { + "fieldMask": [], + "kind": "affected", + "recordedAt": "2024-10-20T00:01:00+00:00", + "source": "ics-kaspersky", + "value": "ACME Corp industrial" + } + ], + "statuses": [], + "type": "ics-vendor", + "versionRanges": [ + { + "fixedVersion": null, + "introducedVersion": null, + "lastAffectedVersion": null, + "primitives": { + "evr": null, + "hasVendorExtensions": true, + "nevra": null, + "semVer": null, + "vendorExtensions": { + "ics.vendor": "ACME Corp industrial" + } + }, + "provenance": { + "fieldMask": [], + "kind": "affected", + "recordedAt": "2024-10-20T00:01:00+00:00", + "source": "ics-kaspersky", + "value": "ACME Corp industrial" + }, + "rangeExpression": null, + "rangeKind": "vendor" + } + ] + }, + { + "identifier": "Additional details are provided in CVE", + "platform": null, + "provenance": [ + { + "fieldMask": [], + "kind": "affected", + "recordedAt": "2024-10-20T00:01:00+00:00", + "source": "ics-kaspersky", + "value": "Additional details are provided in CVE" + } + ], + "statuses": [], + "type": "ics-vendor", + "versionRanges": [ + { + "fixedVersion": null, + "introducedVersion": null, + "lastAffectedVersion": null, + "primitives": { + "evr": null, + "hasVendorExtensions": true, + "nevra": null, + "semVer": null, + "vendorExtensions": { + "ics.vendor": "Additional details are provided in CVE" + } + }, + "provenance": { + "fieldMask": [], + "kind": "affected", + "recordedAt": "2024-10-20T00:01:00+00:00", + "source": "ics-kaspersky", + "value": "Additional details are provided in CVE" + }, + "rangeExpression": null, + "rangeKind": "vendor" + } + ] + }, + { + "identifier": "Exploitation of CVE", + "platform": null, + "provenance": [ + { + "fieldMask": [], + "kind": "affected", + "recordedAt": "2024-10-20T00:01:00+00:00", + "source": "ics-kaspersky", + "value": "Exploitation of CVE" + } + ], + "statuses": [], + "type": "ics-vendor", + "versionRanges": [ + { + "fixedVersion": null, + "introducedVersion": null, + "lastAffectedVersion": null, + "primitives": { + "evr": null, + "hasVendorExtensions": true, + "nevra": null, + "semVer": null, + "vendorExtensions": { + "ics.vendor": "Exploitation of CVE" + } + }, + "provenance": { + "fieldMask": [], + "kind": "affected", + "recordedAt": "2024-10-20T00:01:00+00:00", + "source": "ics-kaspersky", + "value": "Exploitation of CVE" + }, + "rangeExpression": null, + "rangeKind": "vendor" + } + ] + }, + { + "identifier": "Vendor", + "platform": null, + "provenance": [ + { + "fieldMask": [], + "kind": "affected", + "recordedAt": "2024-10-20T00:01:00+00:00", + "source": "ics-kaspersky", + "value": "Vendor" + } + ], + "statuses": [], + "type": "ics-vendor", + "versionRanges": [ + { + "fixedVersion": null, + "introducedVersion": null, + "lastAffectedVersion": null, + "primitives": { + "evr": null, + "hasVendorExtensions": true, + "nevra": null, + "semVer": null, + "vendorExtensions": { + "ics.vendor": "Vendor" + } + }, + "provenance": { + "fieldMask": [], + "kind": "affected", + "recordedAt": "2024-10-20T00:01:00+00:00", + "source": "ics-kaspersky", + "value": "Vendor" + }, + "rangeExpression": null, + "rangeKind": "vendor" + } + ] + }, + { + "identifier": "X100, X200", + "platform": null, + "provenance": [ + { + "fieldMask": [], + "kind": "affected", + "recordedAt": "2024-10-20T00:01:00+00:00", + "source": "ics-kaspersky", + "value": "X100, X200" + } + ], + "statuses": [], + "type": "ics-vendor", + "versionRanges": [ + { + "fixedVersion": null, + "introducedVersion": null, + "lastAffectedVersion": null, + "primitives": { + "evr": null, + "hasVendorExtensions": true, + "nevra": null, + "semVer": null, + "vendorExtensions": { + "ics.vendor": "X100, X200" + } + }, + "provenance": { + "fieldMask": [], + "kind": "affected", + "recordedAt": "2024-10-20T00:01:00+00:00", + "source": "ics-kaspersky", + "value": "X100, X200" + }, + "rangeExpression": null, + "rangeKind": "vendor" + } + ] + } + ], + "aliases": [ + "CVE-2024-7777", + "CVE-2024-8888", + "acme-controller-2024" + ], + "cvssMetrics": [], + "exploitKnown": false, + "language": "en", + "modified": "2024-10-15T10:00:00+00:00", + "provenance": [ + { + "fieldMask": [], + "kind": "document", + "recordedAt": "2024-10-20T00:00:00+00:00", + "source": "ics-kaspersky", + "value": "https://ics-cert.example/advisories/acme-controller-2024/" + }, + { + "fieldMask": [], + "kind": "mapping", + "recordedAt": "2024-10-20T00:01:00+00:00", + "source": "ics-kaspersky", + "value": "acme-controller-2024" + } + ], + "published": "2024-10-15T10:00:00+00:00", + "references": [ + { + "kind": "advisory", + "provenance": { + "fieldMask": [], + "kind": "reference", + "recordedAt": "2024-10-20T00:01:00+00:00", + "source": "ics-kaspersky", + "value": "https://ics-cert.example/advisories/acme-controller-2024/" + }, + "sourceTag": "kaspersky-ics", + "summary": null, + "url": "https://ics-cert.example/advisories/acme-controller-2024/" + }, + { + "kind": "advisory", + "provenance": { + "fieldMask": [], + "kind": "reference", + "recordedAt": "2024-10-20T00:01:00+00:00", + "source": "ics-kaspersky", + "value": "https://www.cve.org/CVERecord?id=CVE-2024-7777" + }, + "sourceTag": "CVE-2024-7777", + "summary": null, + "url": "https://www.cve.org/CVERecord?id=CVE-2024-7777" + }, + { + "kind": "advisory", + "provenance": { + "fieldMask": [], + "kind": "reference", + "recordedAt": "2024-10-20T00:01:00+00:00", + "source": "ics-kaspersky", + "value": "https://www.cve.org/CVERecord?id=CVE-2024-8888" + }, + "sourceTag": "CVE-2024-8888", + "summary": null, + "url": "https://www.cve.org/CVERecord?id=CVE-2024-8888" + } + ], + "severity": null, + "summary": "ACME Corp industrial controllers allow remote compromise (CVE-2024-7777).", + "title": "ACME Corp controllers multiple vulnerabilities" +} \ No newline at end of file diff --git a/src/StellaOps.Feedser.Source.Ics.Kaspersky.Tests/Kaspersky/Fixtures/feed-page1.xml b/src/StellaOps.Feedser.Source.Ics.Kaspersky.Tests/Kaspersky/Fixtures/feed-page1.xml new file mode 100644 index 00000000..6a0f5831 --- /dev/null +++ b/src/StellaOps.Feedser.Source.Ics.Kaspersky.Tests/Kaspersky/Fixtures/feed-page1.xml @@ -0,0 +1,17 @@ + + + + Kaspersky ICS CERT - Advisories + https://ics-cert.kaspersky.com/feed-advisories/ + Test feed + + ACME Corp controllers multiple vulnerabilities + https://ics-cert.example/advisories/acme-controller-2024/ + + Tue, 15 Oct 2024 10:00:00 +0000 + Kaspersky ICS CERT + + + diff --git a/src/StellaOps.Feedser.Source.Ics.Kaspersky.Tests/Kaspersky/KasperskyConnectorTests.cs b/src/StellaOps.Feedser.Source.Ics.Kaspersky.Tests/Kaspersky/KasperskyConnectorTests.cs new file mode 100644 index 00000000..9b72d06a --- /dev/null +++ b/src/StellaOps.Feedser.Source.Ics.Kaspersky.Tests/Kaspersky/KasperskyConnectorTests.cs @@ -0,0 +1,346 @@ +using System.Collections.Generic; +using System.Globalization; +using System.IO; +using System.Net; +using System.Net.Http; +using System.Net.Http.Headers; +using System.Text; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Http; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Logging.Abstractions; +using Microsoft.Extensions.Options; +using Microsoft.Extensions.Time.Testing; +using MongoDB.Bson; +using StellaOps.Feedser.Models; +using StellaOps.Feedser.Source.Common.Fetch; +using StellaOps.Feedser.Source.Common.Http; +using StellaOps.Feedser.Source.Common.Testing; +using StellaOps.Feedser.Source.Common; +using StellaOps.Feedser.Source.Ics.Kaspersky; +using StellaOps.Feedser.Source.Ics.Kaspersky.Configuration; +using StellaOps.Feedser.Storage.Mongo; +using StellaOps.Feedser.Storage.Mongo.Advisories; +using StellaOps.Feedser.Storage.Mongo.Documents; +using StellaOps.Feedser.Storage.Mongo.Dtos; +using StellaOps.Feedser.Testing; + +namespace StellaOps.Feedser.Source.Ics.Kaspersky.Tests; + +[Collection("mongo-fixture")] +public sealed class KasperskyConnectorTests : IAsyncLifetime +{ + private readonly MongoIntegrationFixture _fixture; + private readonly FakeTimeProvider _timeProvider; + private readonly CannedHttpMessageHandler _handler; + private ServiceProvider? _serviceProvider; + + public KasperskyConnectorTests(MongoIntegrationFixture fixture) + { + _fixture = fixture; + _timeProvider = new FakeTimeProvider(new DateTimeOffset(2024, 10, 20, 0, 0, 0, TimeSpan.Zero)); + _handler = new CannedHttpMessageHandler(); + } + + [Fact] + public async Task FetchParseMap_CreatesSnapshot() + { + var options = new KasperskyOptions + { + FeedUri = new Uri("https://ics-cert.example/feed-advisories/", UriKind.Absolute), + WindowSize = TimeSpan.FromDays(30), + WindowOverlap = TimeSpan.FromDays(1), + MaxPagesPerFetch = 1, + RequestDelay = TimeSpan.Zero, + }; + + await EnsureServiceProviderAsync(options); + var provider = _serviceProvider!; + + _handler.Clear(); + + _handler.AddTextResponse(options.FeedUri, ReadFixture("feed-page1.xml"), "application/rss+xml"); + var detailUri = new Uri("https://ics-cert.example/advisories/acme-controller-2024/"); + _handler.AddTextResponse(detailUri, ReadFixture("detail-acme-controller-2024.html"), "text/html"); + + var connector = new KasperskyConnectorPlugin().Create(provider); + + await connector.FetchAsync(provider, CancellationToken.None); + + _timeProvider.Advance(TimeSpan.FromMinutes(1)); + await connector.ParseAsync(provider, CancellationToken.None); + + await connector.MapAsync(provider, CancellationToken.None); + + var advisoryStore = provider.GetRequiredService(); + var advisories = await advisoryStore.GetRecentAsync(5, CancellationToken.None); + Assert.Single(advisories); + var canonical = SnapshotSerializer.ToSnapshot(advisories.Single()); + var expected = ReadFixture("expected-advisory.json"); + var normalizedExpected = NormalizeLineEndings(expected); + var normalizedActual = NormalizeLineEndings(canonical); + if (!string.Equals(normalizedExpected, normalizedActual, StringComparison.Ordinal)) + { + var actualPath = Path.Combine(AppContext.BaseDirectory, "Source", "Ics", "Kaspersky", "Fixtures", "expected-advisory.actual.json"); + Directory.CreateDirectory(Path.GetDirectoryName(actualPath)!); + File.WriteAllText(actualPath, canonical); + } + + Assert.Equal(normalizedExpected, normalizedActual); + + var documentStore = provider.GetRequiredService(); + var document = await documentStore.FindBySourceAndUriAsync(KasperskyConnectorPlugin.SourceName, detailUri.ToString(), CancellationToken.None); + Assert.NotNull(document); + Assert.Equal(DocumentStatuses.Mapped, document!.Status); + + var stateRepository = provider.GetRequiredService(); + var state = await stateRepository.TryGetAsync(KasperskyConnectorPlugin.SourceName, CancellationToken.None); + Assert.NotNull(state); + var pendingDocuments = state!.Cursor.TryGetValue("pendingDocuments", out var pending) + ? pending.AsBsonArray + : new BsonArray(); + Assert.Empty(pendingDocuments); + } + + [Fact] + public async Task FetchFailure_RecordsBackoff() + { + var options = new KasperskyOptions + { + FeedUri = new Uri("https://ics-cert.example/feed-advisories/", UriKind.Absolute), + WindowSize = TimeSpan.FromDays(30), + WindowOverlap = TimeSpan.FromDays(1), + MaxPagesPerFetch = 1, + RequestDelay = TimeSpan.Zero, + }; + + await EnsureServiceProviderAsync(options); + var provider = _serviceProvider!; + _handler.Clear(); + _handler.AddResponse(options.FeedUri, () => new HttpResponseMessage(HttpStatusCode.InternalServerError) + { + Content = new StringContent("feed error", Encoding.UTF8, "text/plain"), + }); + + var connector = new KasperskyConnectorPlugin().Create(provider); + + await Assert.ThrowsAsync(() => connector.FetchAsync(provider, CancellationToken.None)); + + var stateRepository = provider.GetRequiredService(); + var state = await stateRepository.TryGetAsync(KasperskyConnectorPlugin.SourceName, CancellationToken.None); + Assert.NotNull(state); + Assert.Equal(1, state!.FailCount); + Assert.NotNull(state.LastFailureReason); + Assert.Contains("500", state.LastFailureReason, StringComparison.Ordinal); + Assert.True(state.BackoffUntil.HasValue); + Assert.True(state.BackoffUntil!.Value > _timeProvider.GetUtcNow()); + } + + [Fact] + public async Task Fetch_NotModifiedMaintainsDocumentState() + { + var options = new KasperskyOptions + { + FeedUri = new Uri("https://ics-cert.example/feed-advisories/", UriKind.Absolute), + WindowSize = TimeSpan.FromDays(30), + WindowOverlap = TimeSpan.FromDays(1), + MaxPagesPerFetch = 1, + RequestDelay = TimeSpan.Zero, + }; + + await EnsureServiceProviderAsync(options); + var provider = _serviceProvider!; + _handler.Clear(); + + var feedXml = ReadFixture("feed-page1.xml"); + var detailUri = new Uri("https://ics-cert.example/advisories/acme-controller-2024/"); + var detailHtml = ReadFixture("detail-acme-controller-2024.html"); + var etag = new EntityTagHeaderValue("\"ics-2024-acme\""); + var lastModified = new DateTimeOffset(2024, 10, 15, 10, 0, 0, TimeSpan.Zero); + + _handler.AddTextResponse(options.FeedUri, feedXml, "application/rss+xml"); + _handler.AddResponse(detailUri, () => + { + var response = new HttpResponseMessage(HttpStatusCode.OK) + { + Content = new StringContent(detailHtml, Encoding.UTF8, "text/html"), + }; + response.Headers.ETag = etag; + response.Content.Headers.LastModified = lastModified; + return response; + }); + + var connector = new KasperskyConnectorPlugin().Create(provider); + + await connector.FetchAsync(provider, CancellationToken.None); + _timeProvider.Advance(TimeSpan.FromMinutes(1)); + await connector.ParseAsync(provider, CancellationToken.None); + await connector.MapAsync(provider, CancellationToken.None); + + var documentStore = provider.GetRequiredService(); + var document = await documentStore.FindBySourceAndUriAsync(KasperskyConnectorPlugin.SourceName, detailUri.ToString(), CancellationToken.None); + Assert.NotNull(document); + Assert.Equal(DocumentStatuses.Mapped, document!.Status); + + _handler.AddTextResponse(options.FeedUri, feedXml, "application/rss+xml"); + _handler.AddResponse(detailUri, () => + { + var response = new HttpResponseMessage(HttpStatusCode.NotModified); + response.Headers.ETag = etag; + return response; + }); + + await connector.FetchAsync(provider, CancellationToken.None); + await connector.ParseAsync(provider, CancellationToken.None); + await connector.MapAsync(provider, CancellationToken.None); + + document = await documentStore.FindBySourceAndUriAsync(KasperskyConnectorPlugin.SourceName, detailUri.ToString(), CancellationToken.None); + Assert.NotNull(document); + Assert.Equal(DocumentStatuses.Mapped, document!.Status); + + var stateRepository = provider.GetRequiredService(); + var state = await stateRepository.TryGetAsync(KasperskyConnectorPlugin.SourceName, CancellationToken.None); + Assert.NotNull(state); + Assert.True(state!.Cursor.TryGetValue("pendingDocuments", out var pendingDocs)); + Assert.Equal(0, pendingDocs.AsBsonArray.Count); + Assert.True(state.Cursor.TryGetValue("pendingMappings", out var pendingMappings)); + Assert.Equal(0, pendingMappings.AsBsonArray.Count); + } + + [Fact] + public async Task Fetch_DuplicateContentSkipsRequeue() + { + var options = new KasperskyOptions + { + FeedUri = new Uri("https://ics-cert.example/feed-advisories/", UriKind.Absolute), + WindowSize = TimeSpan.FromDays(30), + WindowOverlap = TimeSpan.FromDays(1), + MaxPagesPerFetch = 1, + RequestDelay = TimeSpan.Zero, + }; + + await EnsureServiceProviderAsync(options); + var provider = _serviceProvider!; + _handler.Clear(); + + var feedXml = ReadFixture("feed-page1.xml"); + var detailUri = new Uri("https://ics-cert.example/advisories/acme-controller-2024/"); + var detailHtml = ReadFixture("detail-acme-controller-2024.html"); + + _handler.AddTextResponse(options.FeedUri, feedXml, "application/rss+xml"); + _handler.AddTextResponse(detailUri, detailHtml, "text/html"); + + var connector = new KasperskyConnectorPlugin().Create(provider); + + await connector.FetchAsync(provider, CancellationToken.None); + _timeProvider.Advance(TimeSpan.FromMinutes(1)); + await connector.ParseAsync(provider, CancellationToken.None); + await connector.MapAsync(provider, CancellationToken.None); + + var documentStore = provider.GetRequiredService(); + var document = await documentStore.FindBySourceAndUriAsync(KasperskyConnectorPlugin.SourceName, detailUri.ToString(), CancellationToken.None); + Assert.NotNull(document); + Assert.Equal(DocumentStatuses.Mapped, document!.Status); + + _handler.AddTextResponse(options.FeedUri, feedXml, "application/rss+xml"); + _handler.AddTextResponse(detailUri, detailHtml, "text/html"); + + await connector.FetchAsync(provider, CancellationToken.None); + await connector.ParseAsync(provider, CancellationToken.None); + await connector.MapAsync(provider, CancellationToken.None); + + document = await documentStore.FindBySourceAndUriAsync(KasperskyConnectorPlugin.SourceName, detailUri.ToString(), CancellationToken.None); + Assert.NotNull(document); + Assert.Equal(DocumentStatuses.Mapped, document!.Status); + + var stateRepository = provider.GetRequiredService(); + var state = await stateRepository.TryGetAsync(KasperskyConnectorPlugin.SourceName, CancellationToken.None); + Assert.NotNull(state); + var pendingDocs = state!.Cursor.TryGetValue("pendingDocuments", out var pendingDocsValue) + ? pendingDocsValue.AsBsonArray + : new BsonArray(); + Assert.Empty(pendingDocs); + var pendingMappings = state.Cursor.TryGetValue("pendingMappings", out var pendingMappingsValue) + ? pendingMappingsValue.AsBsonArray + : new BsonArray(); + Assert.Empty(pendingMappings); + } + + private async Task EnsureServiceProviderAsync(KasperskyOptions template) + { + if (_serviceProvider is not null) + { + await ResetDatabaseAsync(); + return; + } + + await _fixture.Client.DropDatabaseAsync(_fixture.Database.DatabaseNamespace.DatabaseName); + + var services = new ServiceCollection(); + services.AddLogging(builder => builder.AddProvider(NullLoggerProvider.Instance)); + services.AddSingleton(_timeProvider); + services.AddSingleton(_handler); + + services.AddMongoStorage(options => + { + options.ConnectionString = _fixture.Runner.ConnectionString; + options.DatabaseName = _fixture.Database.DatabaseNamespace.DatabaseName; + options.CommandTimeout = TimeSpan.FromSeconds(5); + }); + + services.AddSourceCommon(); + services.AddKasperskyIcsConnector(opts => + { + opts.FeedUri = template.FeedUri; + opts.WindowSize = template.WindowSize; + opts.WindowOverlap = template.WindowOverlap; + opts.MaxPagesPerFetch = template.MaxPagesPerFetch; + opts.RequestDelay = template.RequestDelay; + }); + + services.Configure(KasperskyOptions.HttpClientName, builderOptions => + { + builderOptions.HttpMessageHandlerBuilderActions.Add(builder => + { + builder.PrimaryHandler = _handler; + }); + }); + + _serviceProvider = services.BuildServiceProvider(); + var bootstrapper = _serviceProvider.GetRequiredService(); + await bootstrapper.InitializeAsync(CancellationToken.None); + } + + private Task ResetDatabaseAsync() + => _fixture.Client.DropDatabaseAsync(_fixture.Database.DatabaseNamespace.DatabaseName); + + private static string ReadFixture(string filename) + { + var baseDirectory = AppContext.BaseDirectory; + var primary = Path.Combine(baseDirectory, "Source", "Ics", "Kaspersky", "Fixtures", filename); + if (File.Exists(primary)) + { + return File.ReadAllText(primary); + } + + var fallback = Path.Combine(baseDirectory, "Kaspersky", "Fixtures", filename); + return File.ReadAllText(fallback); + } + + private static string NormalizeLineEndings(string value) + => value.Replace("\r\n", "\n", StringComparison.Ordinal); + + public Task InitializeAsync() => Task.CompletedTask; + + public async Task DisposeAsync() + { + if (_serviceProvider is IAsyncDisposable asyncDisposable) + { + await asyncDisposable.DisposeAsync(); + } + else + { + _serviceProvider?.Dispose(); + } + } +} diff --git a/src/StellaOps.Feedser.Source.Ics.Kaspersky.Tests/StellaOps.Feedser.Source.Ics.Kaspersky.Tests.csproj b/src/StellaOps.Feedser.Source.Ics.Kaspersky.Tests/StellaOps.Feedser.Source.Ics.Kaspersky.Tests.csproj new file mode 100644 index 00000000..c3a057e2 --- /dev/null +++ b/src/StellaOps.Feedser.Source.Ics.Kaspersky.Tests/StellaOps.Feedser.Source.Ics.Kaspersky.Tests.csproj @@ -0,0 +1,16 @@ + + + net10.0 + enable + enable + + + + + + + + + + + diff --git a/src/StellaOps.Feedser.Source.Ics.Kaspersky/AGENTS.md b/src/StellaOps.Feedser.Source.Ics.Kaspersky/AGENTS.md new file mode 100644 index 00000000..f76285ff --- /dev/null +++ b/src/StellaOps.Feedser.Source.Ics.Kaspersky/AGENTS.md @@ -0,0 +1,28 @@ +# AGENTS +## Role +Kaspersky ICS-CERT connector; authoritative for OT/ICS vendor advisories covered by Kaspersky ICS-CERT; maps affected products as ICS domain entities with platform tags. +## Scope +- Discover/fetch advisories list; window by publish date or slug; fetch detail pages; handle pagination. +- Validate HTML or JSON; extract CVEs, affected OT vendors/models/families, mitigations; normalize product taxonomy; map fixed versions if present. +- Persist raw docs with sha256; maintain source_state; idempotent mapping. +## Participants +- Source.Common (HTTP, HTML helpers, validators). +- Storage.Mongo (document, dto, advisory, alias, affected, reference, source_state). +- Models (canonical; affected.platform="ics-vendor", tags for device families). +- Core/WebService (jobs: source:ics-kaspersky:fetch|parse|map). +- Merge engine respects ICS vendor authority for OT impact. +## Interfaces & contracts +- Aliases: CVE ids; if stable ICS-CERT advisory id exists, store scheme "ICS-KASP". +- Affected: Type=vendor; Vendor/Product populated; platforms/tags for device family or firmware line; versions with fixedBy when explicit. +- References: advisory, vendor pages, mitigation guides; typed; deduped. +- Provenance: method=parser; value=advisory slug. +## In/Out of scope +In: ICS advisory mapping, affected vendor products, mitigation references. +Out: firmware downloads; reverse-engineering artifacts. +## Observability & security expectations +- Metrics: SourceDiagnostics publishes `feedser.source.http.*` counters/histograms with `feedser.source=ics-kaspersky` to track fetch totals, parse failures, and mapped affected counts. +- Logs: slugs, vendor/product counts, timing; allowlist host. +## Tests +- Author and review coverage in `../StellaOps.Feedser.Source.Ics.Kaspersky.Tests`. +- Shared fixtures (e.g., `MongoIntegrationFixture`, `ConnectorTestHarness`) live in `../StellaOps.Feedser.Testing`. +- Keep fixtures deterministic; match new cases to real-world advisories or regression scenarios. diff --git a/src/StellaOps.Feedser.Source.Ics.Kaspersky/Configuration/KasperskyOptions.cs b/src/StellaOps.Feedser.Source.Ics.Kaspersky/Configuration/KasperskyOptions.cs new file mode 100644 index 00000000..06fccede --- /dev/null +++ b/src/StellaOps.Feedser.Source.Ics.Kaspersky/Configuration/KasperskyOptions.cs @@ -0,0 +1,53 @@ +using System; +using System.Diagnostics.CodeAnalysis; + +namespace StellaOps.Feedser.Source.Ics.Kaspersky.Configuration; + +public sealed class KasperskyOptions +{ + public static string HttpClientName => "source.ics.kaspersky"; + + public Uri FeedUri { get; set; } = new("https://ics-cert.kaspersky.com/feed-advisories/", UriKind.Absolute); + + public TimeSpan WindowSize { get; set; } = TimeSpan.FromDays(30); + + public TimeSpan WindowOverlap { get; set; } = TimeSpan.FromDays(2); + + public int MaxPagesPerFetch { get; set; } = 3; + + public TimeSpan RequestDelay { get; set; } = TimeSpan.FromMilliseconds(500); + + [MemberNotNull(nameof(FeedUri))] + public void Validate() + { + if (FeedUri is null || !FeedUri.IsAbsoluteUri) + { + throw new InvalidOperationException("FeedUri must be an absolute URI."); + } + + if (WindowSize <= TimeSpan.Zero) + { + throw new InvalidOperationException("WindowSize must be greater than zero."); + } + + if (WindowOverlap < TimeSpan.Zero) + { + throw new InvalidOperationException("WindowOverlap cannot be negative."); + } + + if (WindowOverlap >= WindowSize) + { + throw new InvalidOperationException("WindowOverlap must be smaller than WindowSize."); + } + + if (MaxPagesPerFetch <= 0) + { + throw new InvalidOperationException("MaxPagesPerFetch must be positive."); + } + + if (RequestDelay < TimeSpan.Zero) + { + throw new InvalidOperationException("RequestDelay cannot be negative."); + } + } +} diff --git a/src/StellaOps.Feedser.Source.Ics.Kaspersky/Internal/KasperskyAdvisoryDto.cs b/src/StellaOps.Feedser.Source.Ics.Kaspersky/Internal/KasperskyAdvisoryDto.cs new file mode 100644 index 00000000..db2f6ae4 --- /dev/null +++ b/src/StellaOps.Feedser.Source.Ics.Kaspersky/Internal/KasperskyAdvisoryDto.cs @@ -0,0 +1,14 @@ +using System; +using System.Collections.Immutable; + +namespace StellaOps.Feedser.Source.Ics.Kaspersky.Internal; + +internal sealed record KasperskyAdvisoryDto( + string AdvisoryKey, + string Title, + string Link, + DateTimeOffset Published, + string? Summary, + string Content, + ImmutableArray CveIds, + ImmutableArray VendorNames); diff --git a/src/StellaOps.Feedser.Source.Ics.Kaspersky/Internal/KasperskyAdvisoryParser.cs b/src/StellaOps.Feedser.Source.Ics.Kaspersky/Internal/KasperskyAdvisoryParser.cs new file mode 100644 index 00000000..0a3d5419 --- /dev/null +++ b/src/StellaOps.Feedser.Source.Ics.Kaspersky/Internal/KasperskyAdvisoryParser.cs @@ -0,0 +1,172 @@ +using System; +using System.Collections.Generic; +using System.Collections.Immutable; +using System.Linq; +using System.Text; +using System.Text.RegularExpressions; +using System.Threading.Tasks; + +namespace StellaOps.Feedser.Source.Ics.Kaspersky.Internal; + +internal static class KasperskyAdvisoryParser +{ + private static readonly Regex CveRegex = new("CVE-\\d{4}-\\d+", RegexOptions.IgnoreCase | RegexOptions.Compiled); + private static readonly Regex WhitespaceRegex = new("\\s+", RegexOptions.Compiled); + + public static KasperskyAdvisoryDto Parse( + string advisoryKey, + string title, + string link, + DateTimeOffset published, + string? summary, + byte[] rawHtml) + { + var content = ExtractText(rawHtml); + var cves = ExtractCves(title, summary, content); + var vendors = ExtractVendors(title, summary, content); + + return new KasperskyAdvisoryDto( + advisoryKey, + title, + link, + published, + summary, + content, + cves, + vendors); + } + + private static string ExtractText(byte[] rawHtml) + { + if (rawHtml.Length == 0) + { + return string.Empty; + } + + var html = Encoding.UTF8.GetString(rawHtml); + html = Regex.Replace(html, "", string.Empty, RegexOptions.IgnoreCase); + html = Regex.Replace(html, "", string.Empty, RegexOptions.IgnoreCase); + html = Regex.Replace(html, "", string.Empty, RegexOptions.Singleline); + html = Regex.Replace(html, "<[^>]+>", " "); + var decoded = System.Net.WebUtility.HtmlDecode(html); + return string.IsNullOrWhiteSpace(decoded) ? string.Empty : WhitespaceRegex.Replace(decoded, " ").Trim(); + } + + private static ImmutableArray ExtractCves(string title, string? summary, string content) + { + var set = new HashSet(StringComparer.OrdinalIgnoreCase); + void Capture(string? text) + { + if (string.IsNullOrWhiteSpace(text)) + { + return; + } + + foreach (Match match in CveRegex.Matches(text)) + { + if (match.Success) + { + set.Add(match.Value.ToUpperInvariant()); + } + } + } + + Capture(title); + Capture(summary); + Capture(content); + + return set.OrderBy(static cve => cve, StringComparer.Ordinal).ToImmutableArray(); + } + + private static ImmutableArray ExtractVendors(string title, string? summary, string content) + { + var candidates = new HashSet(StringComparer.OrdinalIgnoreCase); + + void AddCandidate(string? text) + { + if (string.IsNullOrWhiteSpace(text)) + { + return; + } + + foreach (var segment in SplitSegments(text)) + { + var cleaned = CleanVendorSegment(segment); + if (!string.IsNullOrWhiteSpace(cleaned)) + { + candidates.Add(cleaned); + } + } + } + + AddCandidate(title); + AddCandidate(summary); + AddCandidate(content); + + return candidates.Count == 0 + ? ImmutableArray.Empty + : candidates + .OrderBy(static vendor => vendor, StringComparer.Ordinal) + .ToImmutableArray(); + } + + private static IEnumerable SplitSegments(string text) + { + var separators = new[] { ".", "-", "–", "—", ":" }; + var queue = new Queue(); + queue.Enqueue(text); + + foreach (var separator in separators) + { + var count = queue.Count; + for (var i = 0; i < count; i++) + { + var item = queue.Dequeue(); + var parts = item.Split(separator, StringSplitOptions.TrimEntries | StringSplitOptions.RemoveEmptyEntries); + foreach (var part in parts) + { + queue.Enqueue(part); + } + } + } + + return queue; + } + + private static string? CleanVendorSegment(string value) + { + var trimmed = value.Trim(); + if (string.IsNullOrEmpty(trimmed)) + { + return null; + } + + var lowered = trimmed.ToLowerInvariant(); + if (lowered.Contains("cve-", StringComparison.Ordinal) || lowered.Contains("vulnerability", StringComparison.Ordinal)) + { + trimmed = trimmed.Split(new[] { "vulnerability", "vulnerabilities" }, StringSplitOptions.RemoveEmptyEntries | StringSplitOptions.TrimEntries).FirstOrDefault() ?? trimmed; + } + + var providedMatch = Regex.Match(trimmed, "provided by\\s+(?[A-Za-z0-9&.,' ]+)", RegexOptions.IgnoreCase); + if (providedMatch.Success) + { + trimmed = providedMatch.Groups["vendor"].Value; + } + + var descriptorMatch = Regex.Match(trimmed, "^(?[A-Z][A-Za-z0-9&.,' ]{1,80}?)(?:\\s+(?:controllers?|devices?|modules?|products?|gateways?|routers?|appliances?|systems?|solutions?|firmware))\\b", RegexOptions.IgnoreCase); + if (descriptorMatch.Success) + { + trimmed = descriptorMatch.Groups["vendor"].Value; + } + + trimmed = trimmed.Replace("’", "'", StringComparison.Ordinal); + trimmed = trimmed.Replace("\"", string.Empty, StringComparison.Ordinal); + + if (trimmed.Length > 200) + { + trimmed = trimmed[..200]; + } + + return string.IsNullOrWhiteSpace(trimmed) ? null : trimmed; + } +} diff --git a/src/StellaOps.Feedser.Source.Ics.Kaspersky/Internal/KasperskyCursor.cs b/src/StellaOps.Feedser.Source.Ics.Kaspersky/Internal/KasperskyCursor.cs new file mode 100644 index 00000000..74ff6fad --- /dev/null +++ b/src/StellaOps.Feedser.Source.Ics.Kaspersky/Internal/KasperskyCursor.cs @@ -0,0 +1,207 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using MongoDB.Bson; + +namespace StellaOps.Feedser.Source.Ics.Kaspersky.Internal; + +internal sealed record KasperskyCursor( + DateTimeOffset? LastPublished, + IReadOnlyCollection PendingDocuments, + IReadOnlyCollection PendingMappings, + IReadOnlyDictionary FetchCache) +{ + private static readonly IReadOnlyCollection EmptyGuidList = Array.Empty(); + private static readonly IReadOnlyDictionary EmptyFetchCache = + new Dictionary(StringComparer.OrdinalIgnoreCase); + + public static KasperskyCursor Empty { get; } = new(null, EmptyGuidList, EmptyGuidList, EmptyFetchCache); + + public BsonDocument ToBsonDocument() + { + var document = new BsonDocument + { + ["pendingDocuments"] = new BsonArray(PendingDocuments.Select(id => id.ToString())), + ["pendingMappings"] = new BsonArray(PendingMappings.Select(id => id.ToString())), + }; + + if (LastPublished.HasValue) + { + document["lastPublished"] = LastPublished.Value.UtcDateTime; + } + + if (FetchCache.Count > 0) + { + var cacheArray = new BsonArray(); + foreach (var (uri, metadata) in FetchCache) + { + var cacheDocument = new BsonDocument + { + ["uri"] = uri, + }; + + if (!string.IsNullOrWhiteSpace(metadata.ETag)) + { + cacheDocument["etag"] = metadata.ETag; + } + + if (metadata.LastModified.HasValue) + { + cacheDocument["lastModified"] = metadata.LastModified.Value.UtcDateTime; + } + + cacheArray.Add(cacheDocument); + } + + document["fetchCache"] = cacheArray; + } + + return document; + } + + public static KasperskyCursor FromBson(BsonDocument? document) + { + if (document is null || document.ElementCount == 0) + { + return Empty; + } + + var lastPublished = document.TryGetValue("lastPublished", out var lastPublishedValue) + ? ParseDate(lastPublishedValue) + : null; + + var pendingDocuments = ReadGuidArray(document, "pendingDocuments"); + var pendingMappings = ReadGuidArray(document, "pendingMappings"); + var fetchCache = ReadFetchCache(document); + + return new KasperskyCursor(lastPublished, pendingDocuments, pendingMappings, fetchCache); + } + + public KasperskyCursor WithPendingDocuments(IEnumerable ids) + => this with { PendingDocuments = ids?.Distinct().ToArray() ?? EmptyGuidList }; + + public KasperskyCursor WithPendingMappings(IEnumerable ids) + => this with { PendingMappings = ids?.Distinct().ToArray() ?? EmptyGuidList }; + + public KasperskyCursor WithLastPublished(DateTimeOffset? timestamp) + => this with { LastPublished = timestamp }; + + public KasperskyCursor WithFetchMetadata(string requestUri, string? etag, DateTimeOffset? lastModified) + { + if (string.IsNullOrWhiteSpace(requestUri)) + { + return this; + } + + var cache = new Dictionary(FetchCache, StringComparer.OrdinalIgnoreCase) + { + [requestUri] = new KasperskyFetchMetadata(etag, lastModified), + }; + + return this with { FetchCache = cache }; + } + + public KasperskyCursor PruneFetchCache(IEnumerable keepUris) + { + if (FetchCache.Count == 0) + { + return this; + } + + var keepSet = new HashSet(keepUris ?? Array.Empty(), StringComparer.OrdinalIgnoreCase); + if (keepSet.Count == 0) + { + return this; + } + + var cache = new Dictionary(StringComparer.OrdinalIgnoreCase); + foreach (var uri in keepSet) + { + if (FetchCache.TryGetValue(uri, out var metadata)) + { + cache[uri] = metadata; + } + } + + return this with { FetchCache = cache }; + } + + public bool TryGetFetchMetadata(string requestUri, out KasperskyFetchMetadata metadata) + { + if (FetchCache.TryGetValue(requestUri, out metadata!)) + { + return true; + } + + metadata = default!; + return false; + } + + private static DateTimeOffset? ParseDate(BsonValue value) + { + return value.BsonType switch + { + BsonType.DateTime => DateTime.SpecifyKind(value.ToUniversalTime(), DateTimeKind.Utc), + BsonType.String when DateTimeOffset.TryParse(value.AsString, out var parsed) => parsed.ToUniversalTime(), + _ => null, + }; + } + + private static IReadOnlyCollection ReadGuidArray(BsonDocument document, string field) + { + if (!document.TryGetValue(field, out var value) || value is not BsonArray array) + { + return Array.Empty(); + } + + var results = new List(array.Count); + foreach (var element in array) + { + if (element is null) + { + continue; + } + + if (Guid.TryParse(element.ToString(), out var guid)) + { + results.Add(guid); + } + } + + return results; + } + + private static IReadOnlyDictionary ReadFetchCache(BsonDocument document) + { + if (!document.TryGetValue("fetchCache", out var value) || value is not BsonArray array) + { + return EmptyFetchCache; + } + + var cache = new Dictionary(StringComparer.OrdinalIgnoreCase); + foreach (var element in array) + { + if (element is not BsonDocument cacheDocument) + { + continue; + } + + if (!cacheDocument.TryGetValue("uri", out var uriValue) || uriValue.BsonType != BsonType.String) + { + continue; + } + + var uri = uriValue.AsString; + string? etag = cacheDocument.TryGetValue("etag", out var etagValue) && etagValue.IsString ? etagValue.AsString : null; + DateTimeOffset? lastModified = cacheDocument.TryGetValue("lastModified", out var lastModifiedValue) + ? ParseDate(lastModifiedValue) + : null; + + cache[uri] = new KasperskyFetchMetadata(etag, lastModified); + } + + return cache.Count == 0 ? EmptyFetchCache : cache; + } +} + +internal sealed record KasperskyFetchMetadata(string? ETag, DateTimeOffset? LastModified); diff --git a/src/StellaOps.Feedser.Source.Ics.Kaspersky/Internal/KasperskyFeedClient.cs b/src/StellaOps.Feedser.Source.Ics.Kaspersky/Internal/KasperskyFeedClient.cs new file mode 100644 index 00000000..efe11249 --- /dev/null +++ b/src/StellaOps.Feedser.Source.Ics.Kaspersky/Internal/KasperskyFeedClient.cs @@ -0,0 +1,133 @@ +using System; +using System.Collections.Generic; +using System.Globalization; +using System.IO; +using System.Linq; +using System.Net.Http; +using System.Text; +using System.Threading; +using System.Threading.Tasks; +using System.Xml.Linq; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Options; +using StellaOps.Feedser.Source.Ics.Kaspersky.Configuration; + +namespace StellaOps.Feedser.Source.Ics.Kaspersky.Internal; + +public sealed class KasperskyFeedClient +{ + private readonly IHttpClientFactory _httpClientFactory; + private readonly KasperskyOptions _options; + private readonly ILogger _logger; + + private static readonly XNamespace ContentNamespace = "http://purl.org/rss/1.0/modules/content/"; + + public KasperskyFeedClient(IHttpClientFactory httpClientFactory, IOptions options, ILogger logger) + { + _httpClientFactory = httpClientFactory ?? throw new ArgumentNullException(nameof(httpClientFactory)); + _options = (options ?? throw new ArgumentNullException(nameof(options))).Value ?? throw new ArgumentNullException(nameof(options)); + _options.Validate(); + _logger = logger ?? throw new ArgumentNullException(nameof(logger)); + } + + public async Task> GetItemsAsync(int page, CancellationToken cancellationToken) + { + var client = _httpClientFactory.CreateClient(KasperskyOptions.HttpClientName); + var feedUri = BuildUri(_options.FeedUri, page); + + using var response = await client.GetAsync(feedUri, cancellationToken).ConfigureAwait(false); + response.EnsureSuccessStatusCode(); + + await using var stream = await response.Content.ReadAsStreamAsync(cancellationToken).ConfigureAwait(false); + using var reader = new StreamReader(stream, Encoding.UTF8); + var xml = await reader.ReadToEndAsync().ConfigureAwait(false); + + var document = XDocument.Parse(xml, LoadOptions.None); + var items = new List(); + var channel = document.Root?.Element("channel"); + if (channel is null) + { + _logger.LogWarning("Feed {FeedUri} is missing channel element", feedUri); + return items; + } + + foreach (var item in channel.Elements("item")) + { + var title = item.Element("title")?.Value?.Trim(); + var linkValue = item.Element("link")?.Value?.Trim(); + var pubDateValue = item.Element("pubDate")?.Value?.Trim(); + var summary = item.Element("description")?.Value?.Trim(); + + if (string.IsNullOrWhiteSpace(title) || string.IsNullOrWhiteSpace(linkValue) || string.IsNullOrWhiteSpace(pubDateValue)) + { + continue; + } + + if (!Uri.TryCreate(linkValue, UriKind.Absolute, out var link)) + { + _logger.LogWarning("Skipping feed item with invalid link: {Link}", linkValue); + continue; + } + + if (!DateTimeOffset.TryParse(pubDateValue, CultureInfo.InvariantCulture, DateTimeStyles.AssumeUniversal, out var published)) + { + _logger.LogWarning("Skipping feed item {Title} due to invalid pubDate {PubDate}", title, pubDateValue); + continue; + } + + var encoded = item.Element(ContentNamespace + "encoded")?.Value; + if (!string.IsNullOrWhiteSpace(encoded)) + { + summary ??= HtmlToPlainText(encoded); + } + + items.Add(new KasperskyFeedItem(title, Canonicalize(link), published.ToUniversalTime(), summary)); + } + + return items; + } + + private static Uri BuildUri(Uri baseUri, int page) + { + if (page <= 1) + { + return baseUri; + } + + var builder = new UriBuilder(baseUri); + var trimmed = builder.Query.TrimStart('?'); + var pageSegment = $"paged={page.ToString(CultureInfo.InvariantCulture)}"; + builder.Query = string.IsNullOrEmpty(trimmed) + ? pageSegment + : $"{trimmed}&{pageSegment}"; + return builder.Uri; + } + + private static Uri Canonicalize(Uri link) + { + if (string.IsNullOrEmpty(link.Query)) + { + return link; + } + + var builder = new UriBuilder(link) + { + Query = string.Empty, + }; + return builder.Uri; + } + + private static string? HtmlToPlainText(string html) + { + if (string.IsNullOrWhiteSpace(html)) + { + return null; + } + + var withoutScripts = System.Text.RegularExpressions.Regex.Replace(html, "", string.Empty, System.Text.RegularExpressions.RegexOptions.IgnoreCase); + var withoutStyles = System.Text.RegularExpressions.Regex.Replace(withoutScripts, "", string.Empty, System.Text.RegularExpressions.RegexOptions.IgnoreCase); + var withoutTags = System.Text.RegularExpressions.Regex.Replace(withoutStyles, "<[^>]+>", " "); + var decoded = System.Net.WebUtility.HtmlDecode(withoutTags); + return string.IsNullOrWhiteSpace(decoded) ? null : System.Text.RegularExpressions.Regex.Replace(decoded, "\\s+", " ").Trim(); + } +} diff --git a/src/StellaOps.Feedser.Source.Ics.Kaspersky/Internal/KasperskyFeedItem.cs b/src/StellaOps.Feedser.Source.Ics.Kaspersky/Internal/KasperskyFeedItem.cs new file mode 100644 index 00000000..eaa552d2 --- /dev/null +++ b/src/StellaOps.Feedser.Source.Ics.Kaspersky/Internal/KasperskyFeedItem.cs @@ -0,0 +1,9 @@ +using System; + +namespace StellaOps.Feedser.Source.Ics.Kaspersky.Internal; + +public sealed record KasperskyFeedItem( + string Title, + Uri Link, + DateTimeOffset Published, + string? Summary); diff --git a/src/StellaOps.Feedser.Source.Ics.Kaspersky/Jobs.cs b/src/StellaOps.Feedser.Source.Ics.Kaspersky/Jobs.cs new file mode 100644 index 00000000..2d752305 --- /dev/null +++ b/src/StellaOps.Feedser.Source.Ics.Kaspersky/Jobs.cs @@ -0,0 +1,46 @@ +using System; +using System.Threading; +using System.Threading.Tasks; +using StellaOps.Feedser.Core.Jobs; + +namespace StellaOps.Feedser.Source.Ics.Kaspersky; + +internal static class KasperskyJobKinds +{ + public const string Fetch = "source:ics-kaspersky:fetch"; + public const string Parse = "source:ics-kaspersky:parse"; + public const string Map = "source:ics-kaspersky:map"; +} + +internal sealed class KasperskyFetchJob : IJob +{ + private readonly KasperskyConnector _connector; + + public KasperskyFetchJob(KasperskyConnector connector) + => _connector = connector ?? throw new ArgumentNullException(nameof(connector)); + + public Task ExecuteAsync(JobExecutionContext context, CancellationToken cancellationToken) + => _connector.FetchAsync(context.Services, cancellationToken); +} + +internal sealed class KasperskyParseJob : IJob +{ + private readonly KasperskyConnector _connector; + + public KasperskyParseJob(KasperskyConnector connector) + => _connector = connector ?? throw new ArgumentNullException(nameof(connector)); + + public Task ExecuteAsync(JobExecutionContext context, CancellationToken cancellationToken) + => _connector.ParseAsync(context.Services, cancellationToken); +} + +internal sealed class KasperskyMapJob : IJob +{ + private readonly KasperskyConnector _connector; + + public KasperskyMapJob(KasperskyConnector connector) + => _connector = connector ?? throw new ArgumentNullException(nameof(connector)); + + public Task ExecuteAsync(JobExecutionContext context, CancellationToken cancellationToken) + => _connector.MapAsync(context.Services, cancellationToken); +} diff --git a/src/StellaOps.Feedser.Source.Ics.Kaspersky/KasperskyConnector.cs b/src/StellaOps.Feedser.Source.Ics.Kaspersky/KasperskyConnector.cs new file mode 100644 index 00000000..727e6821 --- /dev/null +++ b/src/StellaOps.Feedser.Source.Ics.Kaspersky/KasperskyConnector.cs @@ -0,0 +1,464 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text.Json; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Options; +using MongoDB.Bson; +using StellaOps.Feedser.Models; +using StellaOps.Feedser.Source.Common; +using StellaOps.Feedser.Source.Common.Fetch; +using StellaOps.Feedser.Source.Ics.Kaspersky.Configuration; +using StellaOps.Feedser.Source.Ics.Kaspersky.Internal; +using StellaOps.Feedser.Storage.Mongo; +using StellaOps.Feedser.Storage.Mongo.Advisories; +using StellaOps.Feedser.Storage.Mongo.Documents; +using StellaOps.Feedser.Storage.Mongo.Dtos; +using StellaOps.Plugin; + +namespace StellaOps.Feedser.Source.Ics.Kaspersky; + +public sealed class KasperskyConnector : IFeedConnector +{ + private static readonly JsonSerializerOptions SerializerOptions = new(JsonSerializerDefaults.General) + { + PropertyNamingPolicy = JsonNamingPolicy.CamelCase, + WriteIndented = false, + DefaultIgnoreCondition = System.Text.Json.Serialization.JsonIgnoreCondition.WhenWritingNull, + }; + + private readonly KasperskyFeedClient _feedClient; + private readonly SourceFetchService _fetchService; + private readonly RawDocumentStorage _rawDocumentStorage; + private readonly IDocumentStore _documentStore; + private readonly IDtoStore _dtoStore; + private readonly IAdvisoryStore _advisoryStore; + private readonly ISourceStateRepository _stateRepository; + private readonly KasperskyOptions _options; + private readonly TimeProvider _timeProvider; + private readonly ILogger _logger; + + public KasperskyConnector( + KasperskyFeedClient feedClient, + SourceFetchService fetchService, + RawDocumentStorage rawDocumentStorage, + IDocumentStore documentStore, + IDtoStore dtoStore, + IAdvisoryStore advisoryStore, + ISourceStateRepository stateRepository, + IOptions options, + TimeProvider? timeProvider, + ILogger logger) + { + _feedClient = feedClient ?? throw new ArgumentNullException(nameof(feedClient)); + _fetchService = fetchService ?? throw new ArgumentNullException(nameof(fetchService)); + _rawDocumentStorage = rawDocumentStorage ?? throw new ArgumentNullException(nameof(rawDocumentStorage)); + _documentStore = documentStore ?? throw new ArgumentNullException(nameof(documentStore)); + _dtoStore = dtoStore ?? throw new ArgumentNullException(nameof(dtoStore)); + _advisoryStore = advisoryStore ?? throw new ArgumentNullException(nameof(advisoryStore)); + _stateRepository = stateRepository ?? throw new ArgumentNullException(nameof(stateRepository)); + _options = (options ?? throw new ArgumentNullException(nameof(options))).Value ?? throw new ArgumentNullException(nameof(options)); + _options.Validate(); + _timeProvider = timeProvider ?? TimeProvider.System; + _logger = logger ?? throw new ArgumentNullException(nameof(logger)); + } + + public string SourceName => KasperskyConnectorPlugin.SourceName; + + public async Task FetchAsync(IServiceProvider services, CancellationToken cancellationToken) + { + ArgumentNullException.ThrowIfNull(services); + + var cursor = await GetCursorAsync(cancellationToken).ConfigureAwait(false); + var now = _timeProvider.GetUtcNow(); + + var windowStart = cursor.LastPublished.HasValue + ? cursor.LastPublished.Value - _options.WindowOverlap + : now - _options.WindowSize; + + var pendingDocuments = cursor.PendingDocuments.ToHashSet(); + var maxPublished = cursor.LastPublished ?? DateTimeOffset.MinValue; + var cursorState = cursor; + var touchedResources = new HashSet(StringComparer.OrdinalIgnoreCase); + + for (var page = 1; page <= _options.MaxPagesPerFetch; page++) + { + IReadOnlyList items; + try + { + items = await _feedClient.GetItemsAsync(page, cancellationToken).ConfigureAwait(false); + } + catch (Exception ex) + { + _logger.LogError(ex, "Failed to load Kaspersky ICS feed page {Page}", page); + await _stateRepository.MarkFailureAsync( + SourceName, + now, + TimeSpan.FromMinutes(5), + ex.Message, + cancellationToken).ConfigureAwait(false); + throw; + } + if (items.Count == 0) + { + break; + } + + foreach (var item in items) + { + if (item.Published < windowStart) + { + page = _options.MaxPagesPerFetch + 1; + break; + } + + if (_options.RequestDelay > TimeSpan.Zero) + { + await Task.Delay(_options.RequestDelay, cancellationToken).ConfigureAwait(false); + } + + var metadata = new Dictionary(StringComparer.Ordinal) + { + ["kaspersky.title"] = item.Title, + ["kaspersky.link"] = item.Link.ToString(), + ["kaspersky.published"] = item.Published.ToString("O"), + }; + + if (!string.IsNullOrWhiteSpace(item.Summary)) + { + metadata["kaspersky.summary"] = item.Summary!; + } + + var slug = ExtractSlug(item.Link); + if (!string.IsNullOrWhiteSpace(slug)) + { + metadata["kaspersky.slug"] = slug; + } + + var resourceKey = item.Link.ToString(); + touchedResources.Add(resourceKey); + + var existing = await _documentStore.FindBySourceAndUriAsync(SourceName, resourceKey, cancellationToken).ConfigureAwait(false); + + var fetchRequest = new SourceFetchRequest(KasperskyOptions.HttpClientName, SourceName, item.Link) + { + Metadata = metadata, + }; + + if (cursorState.TryGetFetchMetadata(resourceKey, out var cachedFetch)) + { + fetchRequest = fetchRequest with + { + ETag = cachedFetch.ETag, + LastModified = cachedFetch.LastModified, + }; + } + + SourceFetchResult result; + try + { + result = await _fetchService.FetchAsync(fetchRequest, cancellationToken).ConfigureAwait(false); + } + catch (Exception ex) + { + _logger.LogError(ex, "Failed to fetch Kaspersky advisory {Link}", item.Link); + await _stateRepository.MarkFailureAsync( + SourceName, + _timeProvider.GetUtcNow(), + TimeSpan.FromMinutes(5), + ex.Message, + cancellationToken).ConfigureAwait(false); + throw; + } + + if (result.IsNotModified) + { + continue; + } + + if (!result.IsSuccess || result.Document is null) + { + continue; + } + + if (existing is not null + && string.Equals(existing.Sha256, result.Document.Sha256, StringComparison.OrdinalIgnoreCase) + && string.Equals(existing.Status, DocumentStatuses.Mapped, StringComparison.Ordinal)) + { + await _documentStore.UpdateStatusAsync(result.Document.Id, existing.Status, cancellationToken).ConfigureAwait(false); + cursorState = cursorState.WithFetchMetadata(resourceKey, result.Document.Etag, result.Document.LastModified); + if (item.Published > maxPublished) + { + maxPublished = item.Published; + } + + continue; + } + + pendingDocuments.Add(result.Document.Id); + cursorState = cursorState.WithFetchMetadata(resourceKey, result.Document.Etag, result.Document.LastModified); + if (item.Published > maxPublished) + { + maxPublished = item.Published; + } + } + } + + cursorState = cursorState.PruneFetchCache(touchedResources); + + var updatedCursor = cursorState + .WithPendingDocuments(pendingDocuments) + .WithLastPublished(maxPublished == DateTimeOffset.MinValue ? cursor.LastPublished : maxPublished); + + await UpdateCursorAsync(updatedCursor, cancellationToken).ConfigureAwait(false); + } + + public async Task ParseAsync(IServiceProvider services, CancellationToken cancellationToken) + { + ArgumentNullException.ThrowIfNull(services); + + var cursor = await GetCursorAsync(cancellationToken).ConfigureAwait(false); + if (cursor.PendingDocuments.Count == 0) + { + return; + } + + var remainingDocuments = cursor.PendingDocuments.ToList(); + var pendingMappings = cursor.PendingMappings.ToList(); + + foreach (var documentId in cursor.PendingDocuments) + { + cancellationToken.ThrowIfCancellationRequested(); + + var document = await _documentStore.FindAsync(documentId, cancellationToken).ConfigureAwait(false); + if (document is null) + { + remainingDocuments.Remove(documentId); + continue; + } + + if (!document.GridFsId.HasValue) + { + _logger.LogWarning("Kaspersky document {DocumentId} missing GridFS content", document.Id); + await _documentStore.UpdateStatusAsync(document.Id, DocumentStatuses.Failed, cancellationToken).ConfigureAwait(false); + remainingDocuments.Remove(documentId); + continue; + } + + var metadata = document.Metadata ?? new Dictionary(); + var title = metadata.TryGetValue("kaspersky.title", out var titleValue) ? titleValue : document.Uri; + var link = metadata.TryGetValue("kaspersky.link", out var linkValue) ? linkValue : document.Uri; + var published = metadata.TryGetValue("kaspersky.published", out var publishedValue) && DateTimeOffset.TryParse(publishedValue, out var parsedPublished) + ? parsedPublished.ToUniversalTime() + : document.FetchedAt; + var summary = metadata.TryGetValue("kaspersky.summary", out var summaryValue) ? summaryValue : null; + var slug = metadata.TryGetValue("kaspersky.slug", out var slugValue) ? slugValue : ExtractSlug(new Uri(link, UriKind.Absolute)); + var advisoryKey = string.IsNullOrWhiteSpace(slug) ? Guid.NewGuid().ToString("N") : slug; + + byte[] rawBytes; + try + { + rawBytes = await _rawDocumentStorage.DownloadAsync(document.GridFsId.Value, cancellationToken).ConfigureAwait(false); + } + catch (Exception ex) + { + _logger.LogError(ex, "Failed downloading raw Kaspersky document {DocumentId}", document.Id); + throw; + } + + var dto = KasperskyAdvisoryParser.Parse(advisoryKey, title, link, published, summary, rawBytes); + var payload = BsonDocument.Parse(JsonSerializer.Serialize(dto, SerializerOptions)); + var dtoRecord = new DtoRecord(Guid.NewGuid(), document.Id, SourceName, "ics.kaspersky/1", payload, _timeProvider.GetUtcNow()); + + await _dtoStore.UpsertAsync(dtoRecord, cancellationToken).ConfigureAwait(false); + await _documentStore.UpdateStatusAsync(document.Id, DocumentStatuses.PendingMap, cancellationToken).ConfigureAwait(false); + + remainingDocuments.Remove(documentId); + if (!pendingMappings.Contains(documentId)) + { + pendingMappings.Add(documentId); + } + } + + var updatedCursor = cursor + .WithPendingDocuments(remainingDocuments) + .WithPendingMappings(pendingMappings); + + await UpdateCursorAsync(updatedCursor, cancellationToken).ConfigureAwait(false); + } + + public async Task MapAsync(IServiceProvider services, CancellationToken cancellationToken) + { + ArgumentNullException.ThrowIfNull(services); + + var cursor = await GetCursorAsync(cancellationToken).ConfigureAwait(false); + if (cursor.PendingMappings.Count == 0) + { + return; + } + + var pendingMappings = cursor.PendingMappings.ToList(); + + foreach (var documentId in cursor.PendingMappings) + { + cancellationToken.ThrowIfCancellationRequested(); + + var dto = await _dtoStore.FindByDocumentIdAsync(documentId, cancellationToken).ConfigureAwait(false); + var document = await _documentStore.FindAsync(documentId, cancellationToken).ConfigureAwait(false); + + if (dto is null || document is null) + { + _logger.LogWarning("Skipping Kaspersky mapping for {DocumentId}: DTO or document missing", documentId); + pendingMappings.Remove(documentId); + continue; + } + + var dtoJson = dto.Payload.ToJson(new MongoDB.Bson.IO.JsonWriterSettings + { + OutputMode = MongoDB.Bson.IO.JsonOutputMode.RelaxedExtendedJson, + }); + + KasperskyAdvisoryDto advisoryDto; + try + { + advisoryDto = JsonSerializer.Deserialize(dtoJson, SerializerOptions) + ?? throw new InvalidOperationException("Deserialized DTO was null."); + } + catch (Exception ex) + { + _logger.LogError(ex, "Failed to deserialize Kaspersky DTO for {DocumentId}", document.Id); + await _documentStore.UpdateStatusAsync(document.Id, DocumentStatuses.Failed, cancellationToken).ConfigureAwait(false); + pendingMappings.Remove(documentId); + continue; + } + + var fetchProvenance = new AdvisoryProvenance(SourceName, "document", document.Uri, document.FetchedAt); + var mappingProvenance = new AdvisoryProvenance(SourceName, "mapping", advisoryDto.AdvisoryKey, dto.ValidatedAt); + + var aliases = new HashSet(StringComparer.OrdinalIgnoreCase) + { + advisoryDto.AdvisoryKey, + }; + foreach (var cve in advisoryDto.CveIds) + { + aliases.Add(cve); + } + + var references = new List(); + try + { + references.Add(new AdvisoryReference( + advisoryDto.Link, + "advisory", + "kaspersky-ics", + null, + new AdvisoryProvenance(SourceName, "reference", advisoryDto.Link, dto.ValidatedAt))); + } + catch (ArgumentException) + { + _logger.LogWarning("Invalid advisory link {Link} for {AdvisoryKey}", advisoryDto.Link, advisoryDto.AdvisoryKey); + } + + foreach (var cve in advisoryDto.CveIds) + { + var url = $"https://www.cve.org/CVERecord?id={cve}"; + try + { + references.Add(new AdvisoryReference( + url, + "advisory", + cve, + null, + new AdvisoryProvenance(SourceName, "reference", url, dto.ValidatedAt))); + } + catch (ArgumentException) + { + // ignore malformed + } + } + + var affectedPackages = new List(); + foreach (var vendor in advisoryDto.VendorNames) + { + var provenance = new[] + { + new AdvisoryProvenance(SourceName, "affected", vendor, dto.ValidatedAt) + }; + var rangePrimitives = new RangePrimitives( + null, + null, + null, + new Dictionary(StringComparer.OrdinalIgnoreCase) + { + ["ics.vendor"] = vendor + }); + var ranges = new[] + { + new AffectedVersionRange( + rangeKind: "vendor", + introducedVersion: null, + fixedVersion: null, + lastAffectedVersion: null, + rangeExpression: null, + provenance: provenance[0], + primitives: rangePrimitives) + }; + affectedPackages.Add(new AffectedPackage( + AffectedPackageTypes.IcsVendor, + vendor, + platform: null, + versionRanges: ranges, + statuses: Array.Empty(), + provenance: provenance)); + } + + var advisory = new Advisory( + advisoryDto.AdvisoryKey, + advisoryDto.Title, + advisoryDto.Summary ?? advisoryDto.Content, + language: "en", + published: advisoryDto.Published, + modified: advisoryDto.Published, + severity: null, + exploitKnown: false, + aliases: aliases, + references: references, + affectedPackages: affectedPackages, + cvssMetrics: Array.Empty(), + provenance: new[] { fetchProvenance, mappingProvenance }); + + await _advisoryStore.UpsertAsync(advisory, cancellationToken).ConfigureAwait(false); + await _documentStore.UpdateStatusAsync(document.Id, DocumentStatuses.Mapped, cancellationToken).ConfigureAwait(false); + + pendingMappings.Remove(documentId); + } + + var updatedCursor = cursor.WithPendingMappings(pendingMappings); + await UpdateCursorAsync(updatedCursor, cancellationToken).ConfigureAwait(false); + } + + private async Task GetCursorAsync(CancellationToken cancellationToken) + { + var state = await _stateRepository.TryGetAsync(SourceName, cancellationToken).ConfigureAwait(false); + return state is null ? KasperskyCursor.Empty : KasperskyCursor.FromBson(state.Cursor); + } + + private async Task UpdateCursorAsync(KasperskyCursor cursor, CancellationToken cancellationToken) + { + await _stateRepository.UpdateCursorAsync(SourceName, cursor.ToBsonDocument(), _timeProvider.GetUtcNow(), cancellationToken).ConfigureAwait(false); + } + + private static string? ExtractSlug(Uri link) + { + var segments = link.Segments; + if (segments.Length == 0) + { + return null; + } + + var last = segments[^1].Trim('/'); + return string.IsNullOrWhiteSpace(last) && segments.Length > 1 ? segments[^2].Trim('/') : last; + } +} diff --git a/src/StellaOps.Feedser.Source.Ics.Kaspersky/KasperskyConnectorPlugin.cs b/src/StellaOps.Feedser.Source.Ics.Kaspersky/KasperskyConnectorPlugin.cs new file mode 100644 index 00000000..45a05b99 --- /dev/null +++ b/src/StellaOps.Feedser.Source.Ics.Kaspersky/KasperskyConnectorPlugin.cs @@ -0,0 +1,19 @@ +using Microsoft.Extensions.DependencyInjection; +using StellaOps.Plugin; + +namespace StellaOps.Feedser.Source.Ics.Kaspersky; + +public sealed class KasperskyConnectorPlugin : IConnectorPlugin +{ + public const string SourceName = "ics-kaspersky"; + + public string Name => SourceName; + + public bool IsAvailable(IServiceProvider services) => services is not null; + + public IFeedConnector Create(IServiceProvider services) + { + ArgumentNullException.ThrowIfNull(services); + return ActivatorUtilities.CreateInstance(services); + } +} diff --git a/src/StellaOps.Feedser.Source.Ics.Kaspersky/KasperskyDependencyInjectionRoutine.cs b/src/StellaOps.Feedser.Source.Ics.Kaspersky/KasperskyDependencyInjectionRoutine.cs new file mode 100644 index 00000000..a3559421 --- /dev/null +++ b/src/StellaOps.Feedser.Source.Ics.Kaspersky/KasperskyDependencyInjectionRoutine.cs @@ -0,0 +1,54 @@ +using System; +using Microsoft.Extensions.Configuration; +using Microsoft.Extensions.DependencyInjection; +using StellaOps.DependencyInjection; +using StellaOps.Feedser.Core.Jobs; +using StellaOps.Feedser.Source.Ics.Kaspersky.Configuration; + +namespace StellaOps.Feedser.Source.Ics.Kaspersky; + +public sealed class KasperskyDependencyInjectionRoutine : IDependencyInjectionRoutine +{ + private const string ConfigurationSection = "feedser:sources:ics-kaspersky"; + + public IServiceCollection Register(IServiceCollection services, IConfiguration configuration) + { + ArgumentNullException.ThrowIfNull(services); + ArgumentNullException.ThrowIfNull(configuration); + + services.AddKasperskyIcsConnector(options => + { + configuration.GetSection(ConfigurationSection).Bind(options); + options.Validate(); + }); + + services.AddTransient(); + services.AddTransient(); + services.AddTransient(); + + services.PostConfigure(options => + { + EnsureJob(options, KasperskyJobKinds.Fetch, typeof(KasperskyFetchJob)); + EnsureJob(options, KasperskyJobKinds.Parse, typeof(KasperskyParseJob)); + EnsureJob(options, KasperskyJobKinds.Map, typeof(KasperskyMapJob)); + }); + + return services; + } + + private static void EnsureJob(JobSchedulerOptions options, string kind, Type jobType) + { + if (options.Definitions.ContainsKey(kind)) + { + return; + } + + options.Definitions[kind] = new JobDefinition( + kind, + jobType, + options.DefaultTimeout, + options.DefaultLeaseDuration, + CronExpression: null, + Enabled: true); + } +} diff --git a/src/StellaOps.Feedser.Source.Ics.Kaspersky/KasperskyServiceCollectionExtensions.cs b/src/StellaOps.Feedser.Source.Ics.Kaspersky/KasperskyServiceCollectionExtensions.cs new file mode 100644 index 00000000..216fbfaf --- /dev/null +++ b/src/StellaOps.Feedser.Source.Ics.Kaspersky/KasperskyServiceCollectionExtensions.cs @@ -0,0 +1,37 @@ +using System; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Options; +using StellaOps.Feedser.Source.Common.Http; +using StellaOps.Feedser.Source.Ics.Kaspersky.Configuration; +using StellaOps.Feedser.Source.Ics.Kaspersky.Internal; + +namespace StellaOps.Feedser.Source.Ics.Kaspersky; + +public static class KasperskyServiceCollectionExtensions +{ + public static IServiceCollection AddKasperskyIcsConnector(this IServiceCollection services, Action configure) + { + ArgumentNullException.ThrowIfNull(services); + ArgumentNullException.ThrowIfNull(configure); + + services.AddOptions() + .Configure(configure) + .PostConfigure(static opts => opts.Validate()); + + services.AddSourceHttpClient(KasperskyOptions.HttpClientName, (sp, clientOptions) => + { + var options = sp.GetRequiredService>().Value; + clientOptions.BaseAddress = options.FeedUri; + clientOptions.Timeout = TimeSpan.FromSeconds(30); + clientOptions.UserAgent = "StellaOps.Feedser.IcsKaspersky/1.0"; + clientOptions.AllowedHosts.Clear(); + clientOptions.AllowedHosts.Add(options.FeedUri.Host); + clientOptions.DefaultRequestHeaders["Accept"] = "application/rss+xml"; + }); + + services.AddTransient(); + services.AddTransient(); + + return services; + } +} diff --git a/src/StellaOps.Feedser.Source.Ics.Kaspersky/StellaOps.Feedser.Source.Ics.Kaspersky.csproj b/src/StellaOps.Feedser.Source.Ics.Kaspersky/StellaOps.Feedser.Source.Ics.Kaspersky.csproj new file mode 100644 index 00000000..7e54853b --- /dev/null +++ b/src/StellaOps.Feedser.Source.Ics.Kaspersky/StellaOps.Feedser.Source.Ics.Kaspersky.csproj @@ -0,0 +1,16 @@ + + + + net10.0 + enable + enable + + + + + + + + + + diff --git a/src/StellaOps.Feedser.Source.Ics.Kaspersky/TASKS.md b/src/StellaOps.Feedser.Source.Ics.Kaspersky/TASKS.md new file mode 100644 index 00000000..d8df0fcd --- /dev/null +++ b/src/StellaOps.Feedser.Source.Ics.Kaspersky/TASKS.md @@ -0,0 +1,10 @@ +# TASKS +| Task | Owner(s) | Depends on | Notes | +|---|---|---|---| +|List/detail fetcher with windowing|BE-Conn-ICS-Kaspersky|Source.Common|**DONE** – feed client paginates and fetches detail pages with window overlap.| +|Extractor (vendors/models/CVEs)|BE-Conn-ICS-Kaspersky|Source.Common|**DONE** – parser normalizes vendor/model taxonomy into DTO.| +|DTO validation and sanitizer|BE-Conn-ICS-Kaspersky, QA|Source.Common|**DONE** – HTML parsed into DTO with sanitizer guardrails.| +|Canonical mapping (affected, refs)|BE-Conn-ICS-Kaspersky|Models|**DONE** – mapper outputs `ics-vendor` affected entries with provenance.| +|State/dedupe and fixtures|BE-Conn-ICS-Kaspersky, QA|Storage.Mongo|**DONE** – duplicate-content and resume tests exercise SHA gating + cursor hygiene.| +|Backoff on fetch failures|BE-Conn-ICS-Kaspersky|Storage.Mongo|**DONE** – feed/page failures mark source_state with timed backoff.| +|Conditional fetch caching|BE-Conn-ICS-Kaspersky|Source.Common|**DONE** – fetch cache persists ETag/Last-Modified; not-modified scenarios validated in tests.| diff --git a/src/StellaOps.Feedser.Source.Jvn.Tests/Jvn/Fixtures/expected-advisory.json b/src/StellaOps.Feedser.Source.Jvn.Tests/Jvn/Fixtures/expected-advisory.json new file mode 100644 index 00000000..4a691565 --- /dev/null +++ b/src/StellaOps.Feedser.Source.Jvn.Tests/Jvn/Fixtures/expected-advisory.json @@ -0,0 +1,87 @@ +{ + "advisoryKey": "JVNDB-2024-123456", + "affectedPackages": [], + "aliases": [ + "CVE-2024-5555", + "JVNDB-2024-123456" + ], + "cvssMetrics": [ + { + "baseScore": 8.8, + "baseSeverity": "high", + "provenance": { + "fieldMask": [], + "kind": "cvss", + "recordedAt": "2024-03-10T00:01:00+00:00", + "source": "jvn", + "value": "Base" + }, + "vector": "CVSS:3.1/AV:N/AC:L/PR:N/UI:R/S:U/C:H/I:H/A:H", + "version": "3.1" + } + ], + "exploitKnown": false, + "language": "en", + "modified": "2024-03-10T02:30:00+00:00", + "provenance": [ + { + "fieldMask": [], + "kind": "document", + "recordedAt": "2024-03-10T00:00:00+00:00", + "source": "jvn", + "value": "https://jvndb.jvn.jp/myjvn?method=getVulnDetailInfo&feed=hnd&lang=en&vulnId=JVNDB-2024-123456" + }, + { + "fieldMask": [], + "kind": "mapping", + "recordedAt": "2024-03-10T00:01:00+00:00", + "source": "jvn", + "value": "JVNDB-2024-123456" + } + ], + "published": "2024-03-09T02:00:00+00:00", + "references": [ + { + "kind": "weakness", + "provenance": { + "fieldMask": [], + "kind": "reference", + "recordedAt": "2024-03-10T00:01:00+00:00", + "source": "jvn", + "value": "https://cwe.mitre.org/data/definitions/287.html" + }, + "sourceTag": "CWE-287", + "summary": "JVNDB", + "url": "https://cwe.mitre.org/data/definitions/287.html" + }, + { + "kind": "advisory", + "provenance": { + "fieldMask": [], + "kind": "reference", + "recordedAt": "2024-03-10T00:01:00+00:00", + "source": "jvn", + "value": "https://vendor.example.com/advisories/EX-2024-01" + }, + "sourceTag": "EX-2024-01", + "summary": "Example ICS Vendor Advisory", + "url": "https://vendor.example.com/advisories/EX-2024-01" + }, + { + "kind": "advisory", + "provenance": { + "fieldMask": [], + "kind": "reference", + "recordedAt": "2024-03-10T00:01:00+00:00", + "source": "jvn", + "value": "https://www.cve.org/CVERecord?id=CVE-2024-5555" + }, + "sourceTag": "CVE-2024-5555", + "summary": "Common Vulnerabilities and Exposures (CVE)", + "url": "https://www.cve.org/CVERecord?id=CVE-2024-5555" + } + ], + "severity": "high", + "summary": "Imaginary ICS Controller provided by Example Industrial Corporation contains an authentication bypass vulnerability.", + "title": "Example vulnerability in Imaginary ICS Controller" +} \ No newline at end of file diff --git a/src/StellaOps.Feedser.Source.Jvn.Tests/Jvn/Fixtures/jvnrss-window1.xml b/src/StellaOps.Feedser.Source.Jvn.Tests/Jvn/Fixtures/jvnrss-window1.xml new file mode 100644 index 00000000..9027677c --- /dev/null +++ b/src/StellaOps.Feedser.Source.Jvn.Tests/Jvn/Fixtures/jvnrss-window1.xml @@ -0,0 +1,53 @@ + + + + JVNDB Vulnerability countermeasure information + https://jvndb.jvn.jp/apis/myjvn + + 2024-03-10T01:05:00+09:00 + 2024-03-10T01:05:00+09:00 + + + + + + + + Example vulnerability in Imaginary ICS Controller + https://jvndb.jvn.jp/en/contents/2024/JVNDB-2024-123456.html + Sample advisory placeholder. + Information-technology Promotion Agency, Japan + 2024-03-10T01:00:00+09:00 + 2024-03-09T11:00:00+09:00 + 2024-03-10T01:00:00+09:00 + JVNDB-2024-123456 + + + diff --git a/src/StellaOps.Feedser.Source.Jvn.Tests/Jvn/Fixtures/vuldef-JVNDB-2024-123456.xml b/src/StellaOps.Feedser.Source.Jvn.Tests/Jvn/Fixtures/vuldef-JVNDB-2024-123456.xml new file mode 100644 index 00000000..94d149e1 --- /dev/null +++ b/src/StellaOps.Feedser.Source.Jvn.Tests/Jvn/Fixtures/vuldef-JVNDB-2024-123456.xml @@ -0,0 +1,101 @@ + + + + JVNDB-2024-123456 + + Example vulnerability in Imaginary ICS Controller + + Imaginary ICS Controller provided by Example Industrial Corporation contains an authentication bypass vulnerability. + + + + Example Industrial Corporation + Imaginary ICS Controller firmware + cpe:2.3:o:example:imaginary_controller_firmware:2.0 + 2.0.5 + + + + + High + 8.8 + CVSS:3.1/AV:N/AC:L/PR:N/UI:R/S:U/C:H/I:H/A:H + + + A remote attacker could execute arbitrary code. + + + + + Apply firmware version 2.0.6 or later provided by the vendor. + + + + + Example ICS Vendor Advisory + EX-2024-01 + https://vendor.example.com/advisories/EX-2024-01 + + + Vendor advisory duplicate + https://vendor.example.com/advisories/EX-2024-01 + + + Common Vulnerabilities and Exposures (CVE) + CVE-2024-5555 + https://www.cve.org/CVERecord?id=CVE-2024-5555 + + + JVNDB + CWE-287 + Improper Authentication + https://cwe.mitre.org/data/definitions/287.html + + + + + 1 + 2024-03-09T11:00:00+09:00 + [2024/03/09] Initial advisory published. + + + 2 + 2024-03-10T11:30:00+09:00 + [2024/03/10] Vendor solution updated. + + + 2024-03-09T11:00:00+09:00 + 2024-03-10T11:30:00+09:00 + 2024-03-09T00:00:00+09:00 + + + + + + + + + diff --git a/src/StellaOps.Feedser.Source.Jvn.Tests/Jvn/JvnConnectorTests.cs b/src/StellaOps.Feedser.Source.Jvn.Tests/Jvn/JvnConnectorTests.cs new file mode 100644 index 00000000..eafb6cc2 --- /dev/null +++ b/src/StellaOps.Feedser.Source.Jvn.Tests/Jvn/JvnConnectorTests.cs @@ -0,0 +1,311 @@ +using System.Collections.Generic; +using System.Globalization; +using System.IO; +using System.Linq; +using System.Net; +using System.Net.Http; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Http; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Logging.Abstractions; +using Microsoft.Extensions.Options; +using Microsoft.Extensions.Time.Testing; +using MongoDB.Bson; +using MongoDB.Driver; +using StellaOps.Feedser.Models; +using StellaOps.Feedser.Source.Common.Fetch; +using StellaOps.Feedser.Source.Common.Http; +using StellaOps.Feedser.Source.Common.Testing; +using StellaOps.Feedser.Source.Common; +using StellaOps.Feedser.Source.Jvn; +using StellaOps.Feedser.Source.Jvn.Configuration; +using StellaOps.Feedser.Storage.Mongo; +using StellaOps.Feedser.Storage.Mongo.Advisories; +using StellaOps.Feedser.Storage.Mongo.Documents; +using StellaOps.Feedser.Storage.Mongo.Dtos; +using StellaOps.Feedser.Storage.Mongo.JpFlags; +using Xunit.Abstractions; +using StellaOps.Feedser.Testing; + +namespace StellaOps.Feedser.Source.Jvn.Tests; + +[Collection("mongo-fixture")] +public sealed class JvnConnectorTests : IAsyncLifetime +{ + private const string VulnId = "JVNDB-2024-123456"; + + private readonly MongoIntegrationFixture _fixture; + private readonly ITestOutputHelper _output; + private readonly FakeTimeProvider _timeProvider; + private readonly CannedHttpMessageHandler _handler; + private ServiceProvider? _serviceProvider; + + public JvnConnectorTests(MongoIntegrationFixture fixture, ITestOutputHelper output) + { + _fixture = fixture; + _output = output; + _timeProvider = new FakeTimeProvider(new DateTimeOffset(2024, 3, 10, 0, 0, 0, TimeSpan.Zero)); + _handler = new CannedHttpMessageHandler(); + } + + [Fact] + public async Task FetchParseMap_ProducesDeterministicSnapshot() + { + var options = new JvnOptions + { + WindowSize = TimeSpan.FromDays(1), + WindowOverlap = TimeSpan.FromHours(6), + PageSize = 10, + }; + + await EnsureServiceProviderAsync(options); + var provider = _serviceProvider!; + + var now = _timeProvider.GetUtcNow(); + var windowStart = now - options.WindowSize; + var windowEnd = now; + + var overviewUri = BuildOverviewUri(options, windowStart, windowEnd, startItem: 1); + _handler.AddTextResponse(overviewUri, ReadFixture("jvnrss-window1.xml"), "application/xml"); + + var detailUri = BuildDetailUri(options, VulnId); + _handler.AddTextResponse(detailUri, ReadFixture("vuldef-JVNDB-2024-123456.xml"), "application/xml"); + + var connector = new JvnConnectorPlugin().Create(provider); + + await connector.FetchAsync(provider, CancellationToken.None); + + var stateAfterFetch = await provider.GetRequiredService() + .TryGetAsync(JvnConnectorPlugin.SourceName, CancellationToken.None); + if (stateAfterFetch?.Cursor is not null) + { + _output.WriteLine($"Fetch state cursor: {stateAfterFetch.Cursor.ToJson()}"); + } + + var rawDocuments = await _fixture.Database + .GetCollection("document") + .Find(Builders.Filter.Empty) + .ToListAsync(CancellationToken.None); + _output.WriteLine($"Fixture document count: {rawDocuments.Count}"); + + _timeProvider.Advance(TimeSpan.FromMinutes(1)); + await connector.ParseAsync(provider, CancellationToken.None); + + var stateAfterParse = await provider.GetRequiredService() + .TryGetAsync(JvnConnectorPlugin.SourceName, CancellationToken.None); + _output.WriteLine($"Parse state failure reason: {stateAfterParse?.LastFailureReason ?? ""}"); + if (stateAfterParse?.Cursor is not null) + { + _output.WriteLine($"Parse state cursor: {stateAfterParse.Cursor.ToJson()}"); + } + + var dtoCollection = provider.GetRequiredService() + .GetCollection("dto"); + var dtoDocs = await dtoCollection.Find(FilterDefinition.Empty).ToListAsync(CancellationToken.None); + _output.WriteLine($"DTO document count: {dtoDocs.Count}"); + + var documentsAfterParse = await _fixture.Database + .GetCollection("document") + .Find(Builders.Filter.Empty) + .ToListAsync(CancellationToken.None); + _output.WriteLine($"Document statuses after parse: {string.Join(",", documentsAfterParse.Select(d => d.GetValue("status", BsonValue.Create("")).AsString))}"); + + await connector.MapAsync(provider, CancellationToken.None); + + var rawAdvisories = await _fixture.Database + .GetCollection("advisory") + .Find(Builders.Filter.Empty) + .ToListAsync(CancellationToken.None); + _output.WriteLine($"Fixture advisory count: {rawAdvisories.Count}"); + Assert.NotEmpty(rawAdvisories); + + var providerDatabase = provider.GetRequiredService(); + var providerCount = await providerDatabase + .GetCollection("advisory") + .CountDocumentsAsync(FilterDefinition.Empty, cancellationToken: CancellationToken.None); + _output.WriteLine($"Provider advisory count: {providerCount}"); + Assert.True(providerCount > 0, $"Provider DB advisory count was {providerCount}"); + + var typedDocs = await providerDatabase + .GetCollection("advisory") + .Find(FilterDefinition.Empty) + .ToListAsync(CancellationToken.None); + _output.WriteLine($"Typed advisory docs: {typedDocs.Count}"); + Assert.NotEmpty(typedDocs); + + var advisoryStore = provider.GetRequiredService(); + var singleAdvisory = await advisoryStore.FindAsync(VulnId, CancellationToken.None); + Assert.NotNull(singleAdvisory); + _output.WriteLine($"singleAdvisory null? {singleAdvisory is null}"); + + var canonical = SnapshotSerializer.ToSnapshot(singleAdvisory!).Replace("\r\n", "\n"); + var expected = ReadFixture("expected-advisory.json").Replace("\r\n", "\n"); + if (!string.Equals(expected, canonical, StringComparison.Ordinal)) + { + var actualPath = Path.Combine(AppContext.BaseDirectory, "Jvn", "Fixtures", "expected-advisory.actual.json"); + Directory.CreateDirectory(Path.GetDirectoryName(actualPath)!); + File.WriteAllText(actualPath, canonical); + } + Assert.Equal(expected, canonical); + + var jpFlagStore = provider.GetRequiredService(); + var jpFlag = await jpFlagStore.FindAsync(VulnId, CancellationToken.None); + Assert.NotNull(jpFlag); + Assert.Equal("product", jpFlag!.Category); + Assert.Equal("vulnerable", jpFlag.VendorStatus); + + var documentStore = provider.GetRequiredService(); + var document = await documentStore.FindBySourceAndUriAsync(JvnConnectorPlugin.SourceName, detailUri.ToString(), CancellationToken.None); + Assert.NotNull(document); + Assert.Equal(DocumentStatuses.Mapped, document!.Status); + + var stateRepository = provider.GetRequiredService(); + var state = await stateRepository.TryGetAsync(JvnConnectorPlugin.SourceName, CancellationToken.None); + Assert.NotNull(state); + Assert.True(state!.Cursor.TryGetValue("pendingDocuments", out var pendingDocs)); + Assert.Empty(pendingDocs.AsBsonArray); + } + + private async Task EnsureServiceProviderAsync(JvnOptions template) + { + if (_serviceProvider is not null) + { + await ResetDatabaseAsync(); + return; + } + + await _fixture.Client.DropDatabaseAsync(_fixture.Database.DatabaseNamespace.DatabaseName); + + var services = new ServiceCollection(); + services.AddLogging(builder => builder.AddProvider(NullLoggerProvider.Instance)); + services.AddSingleton(_timeProvider); + services.AddSingleton(_handler); + + services.AddMongoStorage(options => + { + options.ConnectionString = _fixture.Runner.ConnectionString; + options.DatabaseName = _fixture.Database.DatabaseNamespace.DatabaseName; + options.CommandTimeout = TimeSpan.FromSeconds(5); + }); + + services.AddSourceCommon(); + services.AddJvnConnector(opts => + { + opts.BaseEndpoint = template.BaseEndpoint; + opts.WindowSize = template.WindowSize; + opts.WindowOverlap = template.WindowOverlap; + opts.PageSize = template.PageSize; + opts.RequestDelay = TimeSpan.Zero; + }); + + services.Configure(JvnOptions.HttpClientName, builderOptions => + { + builderOptions.HttpMessageHandlerBuilderActions.Add(builder => + { + builder.PrimaryHandler = _handler; + }); + }); + + _serviceProvider = services.BuildServiceProvider(); + var bootstrapper = _serviceProvider.GetRequiredService(); + await bootstrapper.InitializeAsync(CancellationToken.None); + } + + private Task ResetDatabaseAsync() + => _fixture.Client.DropDatabaseAsync(_fixture.Database.DatabaseNamespace.DatabaseName); + + private static Uri BuildOverviewUri(JvnOptions options, DateTimeOffset windowStart, DateTimeOffset windowEnd, int startItem) + { + var (startYear, startMonth, startDay) = ToTokyoDateParts(windowStart); + var (endYear, endMonth, endDay) = ToTokyoDateParts(windowEnd); + + var parameters = new List> + { + new("method", "getVulnOverviewList"), + new("feed", "hnd"), + new("lang", "en"), + new("rangeDatePublished", "n"), + new("rangeDatePublic", "n"), + new("rangeDateFirstPublished", "n"), + new("dateFirstPublishedStartY", startYear), + new("dateFirstPublishedStartM", startMonth), + new("dateFirstPublishedStartD", startDay), + new("dateFirstPublishedEndY", endYear), + new("dateFirstPublishedEndM", endMonth), + new("dateFirstPublishedEndD", endDay), + new("startItem", startItem.ToString(CultureInfo.InvariantCulture)), + new("maxCountItem", options.PageSize.ToString(CultureInfo.InvariantCulture)), + }; + + return BuildUri(options.BaseEndpoint, parameters); + } + + private static Uri BuildDetailUri(JvnOptions options, string vulnId) + { + var parameters = new List> + { + new("method", "getVulnDetailInfo"), + new("feed", "hnd"), + new("lang", "en"), + new("vulnId", vulnId), + }; + + return BuildUri(options.BaseEndpoint, parameters); + } + + private static Uri BuildUri(Uri baseEndpoint, IEnumerable> parameters) + { + var query = string.Join( + "&", + parameters.Select(parameter => + $"{WebUtility.UrlEncode(parameter.Key)}={WebUtility.UrlEncode(parameter.Value)}")); + + var builder = new UriBuilder(baseEndpoint) + { + Query = query, + }; + + return builder.Uri; + } + + private static (string Year, string Month, string Day) ToTokyoDateParts(DateTimeOffset timestamp) + { + var local = timestamp.ToOffset(TimeSpan.FromHours(9)).Date; + return ( + local.Year.ToString("D4", CultureInfo.InvariantCulture), + local.Month.ToString("D2", CultureInfo.InvariantCulture), + local.Day.ToString("D2", CultureInfo.InvariantCulture)); + } + + private static string ReadFixture(string filename) + { + var path = ResolveFixturePath(filename); + return File.ReadAllText(path); + } + + private static string ResolveFixturePath(string filename) + { + var baseDirectory = AppContext.BaseDirectory; + var primary = Path.Combine(baseDirectory, "Source", "Jvn", "Fixtures", filename); + if (File.Exists(primary)) + { + return primary; + } + + return Path.Combine(baseDirectory, "Jvn", "Fixtures", filename); + } + + public Task InitializeAsync() => Task.CompletedTask; + + public async Task DisposeAsync() + { + if (_serviceProvider is IAsyncDisposable asyncDisposable) + { + await asyncDisposable.DisposeAsync(); + } + else + { + _serviceProvider?.Dispose(); + } + } +} diff --git a/src/StellaOps.Feedser.Source.Jvn.Tests/StellaOps.Feedser.Source.Jvn.Tests.csproj b/src/StellaOps.Feedser.Source.Jvn.Tests/StellaOps.Feedser.Source.Jvn.Tests.csproj new file mode 100644 index 00000000..351f26a7 --- /dev/null +++ b/src/StellaOps.Feedser.Source.Jvn.Tests/StellaOps.Feedser.Source.Jvn.Tests.csproj @@ -0,0 +1,16 @@ + + + net10.0 + enable + enable + + + + + + + + + + + diff --git a/src/StellaOps.Feedser.Source.Jvn/AGENTS.md b/src/StellaOps.Feedser.Source.Jvn/AGENTS.md new file mode 100644 index 00000000..8c2234d1 --- /dev/null +++ b/src/StellaOps.Feedser.Source.Jvn/AGENTS.md @@ -0,0 +1,29 @@ +# AGENTS +## Role +Japan JVN/MyJVN connector; national CERT enrichment with strong identifiers (JVNDB) and vendor status; authoritative only where concrete package evidence exists; otherwise enriches text, severity, references, and aliases. +## Scope +- Fetch JVNRSS (overview) and VULDEF (detail) via MyJVN API; window by dateFirstPublished/dateLastUpdated; paginate; respect rate limits. +- Validate XML or JSON payloads; normalize titles, CVEs, JVNDB ids, vendor status, categories; map references and severity text; attach jp_flags. +- Persist raw docs with sha256 and headers; manage source_state cursor; idempotent parse/map. +## Participants +- Source.Common (HTTP, pagination, XML or XSD validators, retries/backoff). +- Storage.Mongo (document, dto, advisory, alias, affected (when concrete), reference, jp_flags, source_state). +- Models (canonical Advisory/Affected/Provenance). +- Core/WebService (jobs: source:jvn:fetch|parse|map). +- Merge engine applies enrichment precedence (does not override distro or PSIRT ranges unless JVN gives explicit package truth). +## Interfaces & contracts +- Aliases include JVNDB-YYYY-NNNNN and CVE ids; scheme "JVNDB". +- jp_flags: { jvndb_id, jvn_category, vendor_status }. +- References typed: advisory/vendor/bulletin; URLs normalized and deduped. +- Affected only when VULDEF gives concrete coordinates; otherwise omit. +- Provenance: method=parser; kind=api; value=endpoint plus query window; recordedAt=fetched time. +## In/Out of scope +In: JVN/MyJVN ingestion, aliases, jp_flags, enrichment mapping, watermarking. +Out: overriding distro or PSIRT ranges without concrete evidence; scraping unofficial mirrors. +## Observability & security expectations +- Metrics: SourceDiagnostics emits `feedser.source.http.*` counters/histograms tagged `feedser.source=jvn`, enabling dashboards to track fetch requests, item counts, parse failures, and enrichment/map activity (including jp_flags) via tag filters. +- Logs: window bounds, jvndb ids processed, vendor_status distribution; redact API keys. +## Tests +- Author and review coverage in `../StellaOps.Feedser.Source.Jvn.Tests`. +- Shared fixtures (e.g., `MongoIntegrationFixture`, `ConnectorTestHarness`) live in `../StellaOps.Feedser.Testing`. +- Keep fixtures deterministic; match new cases to real-world advisories or regression scenarios. diff --git a/src/StellaOps.Feedser.Source.Jvn/Configuration/JvnOptions.cs b/src/StellaOps.Feedser.Source.Jvn/Configuration/JvnOptions.cs new file mode 100644 index 00000000..56a3fa81 --- /dev/null +++ b/src/StellaOps.Feedser.Source.Jvn/Configuration/JvnOptions.cs @@ -0,0 +1,80 @@ +using System.Diagnostics.CodeAnalysis; + +namespace StellaOps.Feedser.Source.Jvn.Configuration; + +/// +/// Options controlling the JVN connector fetch cadence and HTTP client configuration. +/// +public sealed class JvnOptions +{ + public static string HttpClientName => "source.jvn"; + + /// + /// Base endpoint for the MyJVN API. + /// + public Uri BaseEndpoint { get; set; } = new("https://jvndb.jvn.jp/myjvn", UriKind.Absolute); + + /// + /// Size of each fetch window applied to dateFirstPublished/dateLastUpdated queries. + /// + public TimeSpan WindowSize { get; set; } = TimeSpan.FromDays(7); + + /// + /// Overlap applied between consecutive windows to ensure late-arriving updates are captured. + /// + public TimeSpan WindowOverlap { get; set; } = TimeSpan.FromDays(1); + + /// + /// Number of overview records requested per page (MyJVN max is 50). + /// + public int PageSize { get; set; } = 50; + + /// + /// Optional delay enforced between HTTP requests to respect service rate limits. + /// + public TimeSpan RequestDelay { get; set; } = TimeSpan.FromMilliseconds(500); + + /// + /// Maximum number of overview pages the connector will request in a single fetch cycle. + /// + public int MaxOverviewPagesPerFetch { get; set; } = 20; + + [MemberNotNull(nameof(BaseEndpoint))] + public void Validate() + { + if (BaseEndpoint is null || !BaseEndpoint.IsAbsoluteUri) + { + throw new InvalidOperationException("JVN options require an absolute BaseEndpoint."); + } + + if (WindowSize <= TimeSpan.Zero) + { + throw new InvalidOperationException("WindowSize must be greater than zero."); + } + + if (WindowOverlap < TimeSpan.Zero) + { + throw new InvalidOperationException("WindowOverlap cannot be negative."); + } + + if (WindowOverlap >= WindowSize) + { + throw new InvalidOperationException("WindowOverlap must be smaller than WindowSize."); + } + + if (PageSize is < 1 or > 50) + { + throw new InvalidOperationException("PageSize must be between 1 and 50 to satisfy MyJVN limits."); + } + + if (RequestDelay < TimeSpan.Zero) + { + throw new InvalidOperationException("RequestDelay cannot be negative."); + } + + if (MaxOverviewPagesPerFetch <= 0) + { + throw new InvalidOperationException("MaxOverviewPagesPerFetch must be positive."); + } + } +} diff --git a/src/StellaOps.Feedser.Source.Jvn/Internal/JvnAdvisoryMapper.cs b/src/StellaOps.Feedser.Source.Jvn/Internal/JvnAdvisoryMapper.cs new file mode 100644 index 00000000..b6544e86 --- /dev/null +++ b/src/StellaOps.Feedser.Source.Jvn/Internal/JvnAdvisoryMapper.cs @@ -0,0 +1,418 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using StellaOps.Feedser.Models; +using StellaOps.Feedser.Source.Common; +using StellaOps.Feedser.Normalization.Cvss; +using StellaOps.Feedser.Normalization.Identifiers; +using StellaOps.Feedser.Normalization.Text; +using StellaOps.Feedser.Storage.Mongo.Documents; +using StellaOps.Feedser.Storage.Mongo.Dtos; +using StellaOps.Feedser.Storage.Mongo.JpFlags; + +namespace StellaOps.Feedser.Source.Jvn.Internal; + +internal static class JvnAdvisoryMapper +{ + private static readonly string[] SeverityOrder = { "none", "low", "medium", "high", "critical" }; + + public static (Advisory Advisory, JpFlagRecord Flag) Map( + JvnDetailDto detail, + DocumentRecord document, + DtoRecord dtoRecord, + TimeProvider timeProvider) + { + ArgumentNullException.ThrowIfNull(detail); + ArgumentNullException.ThrowIfNull(document); + ArgumentNullException.ThrowIfNull(dtoRecord); + ArgumentNullException.ThrowIfNull(timeProvider); + + var recordedAt = dtoRecord.ValidatedAt; + var fetchProvenance = new AdvisoryProvenance(JvnConnectorPlugin.SourceName, "document", document.Uri, document.FetchedAt); + var mappingProvenance = new AdvisoryProvenance(JvnConnectorPlugin.SourceName, "mapping", detail.VulnerabilityId, recordedAt); + + var aliases = BuildAliases(detail); + var references = BuildReferences(detail, recordedAt); + var affectedPackages = BuildAffected(detail, recordedAt); + var cvssMetrics = BuildCvss(detail, recordedAt, out var severity); + + var description = DescriptionNormalizer.Normalize(new[] + { + new LocalizedText(detail.Overview, detail.Language), + }); + + var language = description.Language; + var summary = string.IsNullOrEmpty(description.Text) ? null : description.Text; + + var provenance = new[] { fetchProvenance, mappingProvenance }; + + var advisory = new Advisory( + detail.VulnerabilityId, + detail.Title, + summary, + language, + detail.DateFirstPublished, + detail.DateLastUpdated, + severity, + exploitKnown: false, + aliases, + references, + affectedPackages, + cvssMetrics, + provenance); + + var vendorStatus = detail.VendorStatuses.Length == 0 + ? null + : string.Join(",", detail.VendorStatuses.OrderBy(static status => status, StringComparer.Ordinal)); + + var flag = new JpFlagRecord( + detail.VulnerabilityId, + JvnConnectorPlugin.SourceName, + detail.JvnCategory, + vendorStatus, + timeProvider.GetUtcNow()); + + return (advisory, flag); + } + + private static IEnumerable BuildAliases(JvnDetailDto detail) + { + var aliases = new HashSet(StringComparer.OrdinalIgnoreCase) + { + detail.VulnerabilityId, + }; + + foreach (var cve in detail.CveIds) + { + if (!string.IsNullOrWhiteSpace(cve)) + { + aliases.Add(cve); + } + } + + return aliases; + } + + private static IEnumerable BuildReferences(JvnDetailDto detail, DateTimeOffset recordedAt) + { + var references = new List(); + + foreach (var reference in detail.References) + { + if (string.IsNullOrWhiteSpace(reference.Url)) + { + continue; + } + + string? kind = reference.Type?.ToLowerInvariant() switch + { + "vendor" => "vendor", + "advisory" => "advisory", + "cwe" => "weakness", + _ => null, + }; + + string? sourceTag = !string.IsNullOrWhiteSpace(reference.Id) ? reference.Id : reference.Type; + string? summary = reference.Name; + + try + { + references.Add(new AdvisoryReference( + reference.Url, + kind, + sourceTag, + summary, + new AdvisoryProvenance(JvnConnectorPlugin.SourceName, "reference", reference.Url, recordedAt))); + } + catch (ArgumentException) + { + // Ignore malformed URLs that slipped through validation. + } + } + + if (references.Count == 0) + { + return references; + } + + var map = new Dictionary(StringComparer.OrdinalIgnoreCase); + foreach (var reference in references) + { + if (!map.TryGetValue(reference.Url, out var existing)) + { + map[reference.Url] = reference; + continue; + } + + map[reference.Url] = MergeReferences(existing, reference); + } + + var deduped = map.Values.ToList(); + deduped.Sort(CompareReferences); + return deduped; + } + + private static IEnumerable BuildAffected(JvnDetailDto detail, DateTimeOffset recordedAt) + { + var packages = new List(); + + foreach (var product in detail.Affected) + { + if (string.IsNullOrWhiteSpace(product.Cpe)) + { + continue; + } + + if (!string.IsNullOrWhiteSpace(product.Status) && !product.Status.StartsWith("vulnerable", StringComparison.OrdinalIgnoreCase)) + { + continue; + } + + if (!IdentifierNormalizer.TryNormalizeCpe(product.Cpe, out var cpe)) + { + continue; + } + + var provenance = new List + { + new AdvisoryProvenance(JvnConnectorPlugin.SourceName, "affected", cpe!, recordedAt), + }; + + var attributeParts = new List(capacity: 2); + if (!string.IsNullOrWhiteSpace(product.CpeVendor)) + { + attributeParts.Add($"vendor={product.CpeVendor}"); + } + + if (!string.IsNullOrWhiteSpace(product.CpeProduct)) + { + attributeParts.Add($"product={product.CpeProduct}"); + } + + if (attributeParts.Count > 0) + { + provenance.Add(new AdvisoryProvenance( + JvnConnectorPlugin.SourceName, + "cpe-attributes", + string.Join(";", attributeParts), + recordedAt)); + } + + var platform = product.Vendor ?? product.CpeVendor; + + var versionRanges = BuildVersionRanges(product, recordedAt, provenance[0]); + + packages.Add(new AffectedPackage( + AffectedPackageTypes.Cpe, + cpe!, + platform: platform, + versionRanges: versionRanges, + statuses: Array.Empty(), + provenance: provenance.ToArray())); + } + + return packages; + } + + private static IReadOnlyList BuildVersionRanges(JvnAffectedProductDto product, DateTimeOffset recordedAt, AdvisoryProvenance provenance) + { + var extensions = new Dictionary(StringComparer.OrdinalIgnoreCase); + if (!string.IsNullOrWhiteSpace(product.Version)) + { + extensions["jvn.version"] = product.Version!; + } + + if (!string.IsNullOrWhiteSpace(product.Build)) + { + extensions["jvn.build"] = product.Build!; + } + + if (!string.IsNullOrWhiteSpace(product.Description)) + { + extensions["jvn.description"] = product.Description!; + } + + if (!string.IsNullOrWhiteSpace(product.Status)) + { + extensions["jvn.status"] = product.Status!; + } + + if (extensions.Count == 0) + { + return Array.Empty(); + } + + var primitives = new RangePrimitives( + null, + null, + null, + extensions); + + var expression = product.Version; + var range = new AffectedVersionRange( + rangeKind: "cpe", + introducedVersion: null, + fixedVersion: null, + lastAffectedVersion: null, + rangeExpression: string.IsNullOrWhiteSpace(expression) ? null : expression, + provenance: provenance, + primitives: primitives); + + return new[] { range }; + } + + private static IReadOnlyList BuildCvss(JvnDetailDto detail, DateTimeOffset recordedAt, out string? severity) + { + var metrics = new List(); + severity = null; + var bestRank = -1; + + foreach (var cvss in detail.Cvss) + { + if (!CvssMetricNormalizer.TryNormalize(cvss.Version, cvss.Vector, cvss.Score, cvss.Severity, out var normalized)) + { + continue; + } + + var provenance = new AdvisoryProvenance(JvnConnectorPlugin.SourceName, "cvss", cvss.Type, recordedAt); + metrics.Add(normalized.ToModel(provenance)); + + var rank = Array.IndexOf(SeverityOrder, normalized.BaseSeverity); + if (rank > bestRank) + { + bestRank = rank; + severity = normalized.BaseSeverity; + } + } + + return metrics; + } + + private static int CompareReferences(AdvisoryReference? left, AdvisoryReference? right) + { + if (ReferenceEquals(left, right)) + { + return 0; + } + + if (left is null) + { + return 1; + } + + if (right is null) + { + return -1; + } + + var compare = StringComparer.OrdinalIgnoreCase.Compare(left.Url, right.Url); + if (compare != 0) + { + return compare; + } + + compare = CompareNullable(left.Kind, right.Kind); + if (compare != 0) + { + return compare; + } + + compare = CompareNullable(left.SourceTag, right.SourceTag); + if (compare != 0) + { + return compare; + } + + compare = CompareNullable(left.Summary, right.Summary); + if (compare != 0) + { + return compare; + } + + compare = StringComparer.Ordinal.Compare(left.Provenance.Source, right.Provenance.Source); + if (compare != 0) + { + return compare; + } + + compare = StringComparer.Ordinal.Compare(left.Provenance.Kind, right.Provenance.Kind); + if (compare != 0) + { + return compare; + } + + compare = CompareNullable(left.Provenance.Value, right.Provenance.Value); + if (compare != 0) + { + return compare; + } + + return left.Provenance.RecordedAt.CompareTo(right.Provenance.RecordedAt); + } + + private static int CompareNullable(string? left, string? right) + { + if (left is null && right is null) + { + return 0; + } + + if (left is null) + { + return 1; + } + + if (right is null) + { + return -1; + } + + return StringComparer.Ordinal.Compare(left, right); + } + + private static AdvisoryReference MergeReferences(AdvisoryReference existing, AdvisoryReference candidate) + { + var kind = existing.Kind ?? candidate.Kind; + var sourceTag = existing.SourceTag ?? candidate.SourceTag; + var summary = ChoosePreferredSummary(existing.Summary, candidate.Summary); + var provenance = existing.Provenance.RecordedAt <= candidate.Provenance.RecordedAt + ? existing.Provenance + : candidate.Provenance; + + if (kind == existing.Kind + && sourceTag == existing.SourceTag + && summary == existing.Summary + && provenance == existing.Provenance) + { + return existing; + } + + if (kind == candidate.Kind + && sourceTag == candidate.SourceTag + && summary == candidate.Summary + && provenance == candidate.Provenance) + { + return candidate; + } + + return new AdvisoryReference(existing.Url, kind, sourceTag, summary, provenance); + } + + private static string? ChoosePreferredSummary(string? left, string? right) + { + var leftValue = string.IsNullOrWhiteSpace(left) ? null : left; + var rightValue = string.IsNullOrWhiteSpace(right) ? null : right; + + if (leftValue is null) + { + return rightValue; + } + + if (rightValue is null) + { + return leftValue; + } + + return leftValue.Length >= rightValue.Length ? leftValue : rightValue; + } +} diff --git a/src/StellaOps.Feedser.Source.Jvn/Internal/JvnConstants.cs b/src/StellaOps.Feedser.Source.Jvn/Internal/JvnConstants.cs new file mode 100644 index 00000000..a3f3fb43 --- /dev/null +++ b/src/StellaOps.Feedser.Source.Jvn/Internal/JvnConstants.cs @@ -0,0 +1,10 @@ +namespace StellaOps.Feedser.Source.Jvn.Internal; + +internal static class JvnConstants +{ + public const string DtoSchemaVersion = "jvn.vuldef.3.2"; + + public const string VuldefNamespace = "http://jvn.jp/vuldef/"; + public const string StatusNamespace = "http://jvndb.jvn.jp/myjvn/Status"; + public const string ModSecNamespace = "http://jvn.jp/rss/mod_sec/3.0/"; +} diff --git a/src/StellaOps.Feedser.Source.Jvn/Internal/JvnCursor.cs b/src/StellaOps.Feedser.Source.Jvn/Internal/JvnCursor.cs new file mode 100644 index 00000000..25be9887 --- /dev/null +++ b/src/StellaOps.Feedser.Source.Jvn/Internal/JvnCursor.cs @@ -0,0 +1,106 @@ +using System.Linq; +using MongoDB.Bson; + +namespace StellaOps.Feedser.Source.Jvn.Internal; + +internal sealed record JvnCursor( + DateTimeOffset? WindowStart, + DateTimeOffset? WindowEnd, + DateTimeOffset? LastCompletedWindowEnd, + IReadOnlyCollection PendingDocuments, + IReadOnlyCollection PendingMappings) +{ + public static JvnCursor Empty { get; } = new(null, null, null, Array.Empty(), Array.Empty()); + + public BsonDocument ToBsonDocument() + { + var document = new BsonDocument(); + + if (WindowStart.HasValue) + { + document["windowStart"] = WindowStart.Value.UtcDateTime; + } + + if (WindowEnd.HasValue) + { + document["windowEnd"] = WindowEnd.Value.UtcDateTime; + } + + if (LastCompletedWindowEnd.HasValue) + { + document["lastCompletedWindowEnd"] = LastCompletedWindowEnd.Value.UtcDateTime; + } + + document["pendingDocuments"] = new BsonArray(PendingDocuments.Select(static id => id.ToString())); + document["pendingMappings"] = new BsonArray(PendingMappings.Select(static id => id.ToString())); + return document; + } + + public static JvnCursor FromBson(BsonDocument? document) + { + if (document is null || document.ElementCount == 0) + { + return Empty; + } + + DateTimeOffset? windowStart = TryGetDateTime(document, "windowStart"); + DateTimeOffset? windowEnd = TryGetDateTime(document, "windowEnd"); + DateTimeOffset? lastCompleted = TryGetDateTime(document, "lastCompletedWindowEnd"); + + var pendingDocuments = ReadGuidArray(document, "pendingDocuments"); + var pendingMappings = ReadGuidArray(document, "pendingMappings"); + + return new JvnCursor(windowStart, windowEnd, lastCompleted, pendingDocuments, pendingMappings); + } + + public JvnCursor WithWindow(DateTimeOffset start, DateTimeOffset end) + => this with { WindowStart = start, WindowEnd = end }; + + public JvnCursor WithCompletedWindow(DateTimeOffset end) + => this with { LastCompletedWindowEnd = end }; + + public JvnCursor WithPendingDocuments(IEnumerable pending) + => this with { PendingDocuments = pending?.Distinct().ToArray() ?? Array.Empty() }; + + public JvnCursor WithPendingMappings(IEnumerable pending) + => this with { PendingMappings = pending?.Distinct().ToArray() ?? Array.Empty() }; + + private static DateTimeOffset? TryGetDateTime(BsonDocument document, string field) + { + if (!document.TryGetValue(field, out var value)) + { + return null; + } + + return value.BsonType switch + { + BsonType.DateTime => DateTime.SpecifyKind(value.ToUniversalTime(), DateTimeKind.Utc), + BsonType.String when DateTimeOffset.TryParse(value.AsString, out var parsed) => parsed.ToUniversalTime(), + _ => null, + }; + } + + private static IReadOnlyCollection ReadGuidArray(BsonDocument document, string field) + { + if (!document.TryGetValue(field, out var value) || value is not BsonArray array) + { + return Array.Empty(); + } + + var results = new List(array.Count); + foreach (var element in array) + { + if (element is null) + { + continue; + } + + if (element.BsonType == BsonType.String && Guid.TryParse(element.AsString, out var guid)) + { + results.Add(guid); + } + } + + return results; + } +} diff --git a/src/StellaOps.Feedser.Source.Jvn/Internal/JvnDetailDto.cs b/src/StellaOps.Feedser.Source.Jvn/Internal/JvnDetailDto.cs new file mode 100644 index 00000000..9e3449bb --- /dev/null +++ b/src/StellaOps.Feedser.Source.Jvn/Internal/JvnDetailDto.cs @@ -0,0 +1,69 @@ +using System.Collections.Immutable; + +namespace StellaOps.Feedser.Source.Jvn.Internal; + +internal sealed record JvnDetailDto( + string VulnerabilityId, + string Title, + string? Overview, + string? Language, + DateTimeOffset? DateFirstPublished, + DateTimeOffset? DateLastUpdated, + DateTimeOffset? DatePublic, + ImmutableArray Cvss, + ImmutableArray Affected, + ImmutableArray References, + ImmutableArray History, + ImmutableArray CweIds, + ImmutableArray CveIds, + string? AdvisoryUrl, + string? JvnCategory, + ImmutableArray VendorStatuses) +{ + public static JvnDetailDto Empty { get; } = new( + "unknown", + "unknown", + null, + null, + null, + null, + null, + ImmutableArray.Empty, + ImmutableArray.Empty, + ImmutableArray.Empty, + ImmutableArray.Empty, + ImmutableArray.Empty, + ImmutableArray.Empty, + null, + null, + ImmutableArray.Empty); +} + +internal sealed record JvnCvssDto( + string Version, + string Type, + string Severity, + double Score, + string? Vector); + +internal sealed record JvnAffectedProductDto( + string? Vendor, + string? Product, + string? Cpe, + string? CpeVendor, + string? CpeProduct, + string? Version, + string? Build, + string? Description, + string? Status); + +internal sealed record JvnReferenceDto( + string Type, + string Id, + string? Name, + string Url); + +internal sealed record JvnHistoryEntryDto( + string? Number, + DateTimeOffset? Timestamp, + string? Description); diff --git a/src/StellaOps.Feedser.Source.Jvn/Internal/JvnDetailParser.cs b/src/StellaOps.Feedser.Source.Jvn/Internal/JvnDetailParser.cs new file mode 100644 index 00000000..24faacb9 --- /dev/null +++ b/src/StellaOps.Feedser.Source.Jvn/Internal/JvnDetailParser.cs @@ -0,0 +1,268 @@ +using System; +using System.Collections.Generic; +using System.Collections.Immutable; +using System.Globalization; +using System.IO; +using System.Linq; +using System.Xml; +using System.Xml.Linq; +using System.Xml.Schema; + +namespace StellaOps.Feedser.Source.Jvn.Internal; + +internal static class JvnDetailParser +{ + private static readonly XNamespace Vuldef = JvnConstants.VuldefNamespace; + private static readonly XNamespace Status = JvnConstants.StatusNamespace; + + public static JvnDetailDto Parse(byte[] payload, string? documentUri) + { + ArgumentNullException.ThrowIfNull(payload); + + using var stream = new MemoryStream(payload, writable: false); + var settings = new XmlReaderSettings + { + DtdProcessing = DtdProcessing.Prohibit, + IgnoreComments = true, + IgnoreProcessingInstructions = true, + IgnoreWhitespace = true, + }; + + using var reader = XmlReader.Create(stream, settings); + var document = XDocument.Load(reader, LoadOptions.None); + Validate(document, documentUri); + return Extract(document, documentUri); + } + + private static void Validate(XDocument document, string? documentUri) + { + void Handler(object? sender, ValidationEventArgs args) + { + throw new JvnSchemaValidationException( + $"JVN schema validation failed for {documentUri ?? ""}: {args.Message}", + args.Exception ?? new XmlSchemaValidationException(args.Message)); + } + + document.Validate(JvnSchemaProvider.SchemaSet, Handler, addSchemaInfo: true); + } + + private static JvnDetailDto Extract(XDocument document, string? documentUri) + { + var root = document.Root ?? throw new InvalidOperationException("JVN VULDEF document missing root element."); + + var vulinfo = root.Element(Vuldef + "Vulinfo") ?? throw new InvalidOperationException("Vulinfo element missing."); + var vulinfoId = Clean(vulinfo.Element(Vuldef + "VulinfoID")?.Value) + ?? throw new InvalidOperationException("VulinfoID element missing."); + + var data = vulinfo.Element(Vuldef + "VulinfoData") ?? throw new InvalidOperationException("VulinfoData element missing."); + var title = Clean(data.Element(Vuldef + "Title")?.Value) ?? vulinfoId; + var overview = Clean(data.Element(Vuldef + "VulinfoDescription")?.Element(Vuldef + "Overview")?.Value); + + var dateFirstPublished = ParseDate(data.Element(Vuldef + "DateFirstPublished")?.Value); + var dateLastUpdated = ParseDate(data.Element(Vuldef + "DateLastUpdated")?.Value); + var datePublic = ParseDate(data.Element(Vuldef + "DatePublic")?.Value); + + var cvssEntries = ParseCvss(data.Element(Vuldef + "Impact")); + var affected = ParseAffected(data.Element(Vuldef + "Affected")); + var references = ParseReferences(data.Element(Vuldef + "Related")); + var history = ParseHistory(data.Element(Vuldef + "History")); + + var cweIds = references.Where(r => string.Equals(r.Type, "cwe", StringComparison.OrdinalIgnoreCase)) + .Select(r => r.Id) + .Where(static id => !string.IsNullOrWhiteSpace(id)) + .Distinct(StringComparer.OrdinalIgnoreCase) + .Select(static id => id!) + .ToImmutableArray(); + + var cveIds = references.Where(r => string.Equals(r.Type, "advisory", StringComparison.OrdinalIgnoreCase) + && !string.IsNullOrWhiteSpace(r.Id) + && r.Id.StartsWith("CVE-", StringComparison.OrdinalIgnoreCase)) + .Select(r => r.Id) + .Where(static id => !string.IsNullOrWhiteSpace(id)) + .Distinct(StringComparer.OrdinalIgnoreCase) + .Select(static id => id!) + .ToImmutableArray(); + + var language = Clean(root.Attribute(XNamespace.Xml + "lang")?.Value); + + var statusElement = root.Element(Status + "Status"); + var jvnCategory = Clean(statusElement?.Attribute("category")?.Value); + + var vendorStatuses = affected + .Select(a => a.Status) + .Where(static status => !string.IsNullOrWhiteSpace(status)) + .Select(static status => status!.ToLowerInvariant()) + .Distinct(StringComparer.Ordinal) + .ToImmutableArray(); + + return new JvnDetailDto( + vulinfoId, + title, + overview, + language, + dateFirstPublished, + dateLastUpdated, + datePublic, + cvssEntries, + affected, + references, + history, + cweIds, + cveIds, + Clean(documentUri), + jvnCategory, + vendorStatuses); + } + + private static ImmutableArray ParseCvss(XElement? impactElement) + { + if (impactElement is null) + { + return ImmutableArray.Empty; + } + + var results = new List(); + foreach (var cvssElement in impactElement.Elements(Vuldef + "Cvss")) + { + var version = Clean(cvssElement.Attribute("version")?.Value) ?? ""; + var severityElement = cvssElement.Element(Vuldef + "Severity"); + var severity = Clean(severityElement?.Value) ?? Clean(cvssElement.Attribute("severity")?.Value) ?? string.Empty; + var type = Clean(severityElement?.Attribute("type")?.Value) ?? Clean(cvssElement.Attribute("type")?.Value) ?? "base"; + var scoreText = Clean(cvssElement.Element(Vuldef + "Base")?.Value) + ?? Clean(cvssElement.Attribute("score")?.Value) + ?? "0"; + if (!double.TryParse(scoreText, NumberStyles.Float, CultureInfo.InvariantCulture, out var score)) + { + score = 0d; + } + + var vector = Clean(cvssElement.Element(Vuldef + "Vector")?.Value) + ?? Clean(cvssElement.Attribute("vector")?.Value); + + results.Add(new JvnCvssDto( + version, + type, + severity, + score, + vector)); + } + + return results.ToImmutableArray(); + } + + private static ImmutableArray ParseAffected(XElement? affectedElement) + { + if (affectedElement is null) + { + return ImmutableArray.Empty; + } + + var results = new List(); + foreach (var item in affectedElement.Elements(Vuldef + "AffectedItem")) + { + var vendor = Clean(item.Element(Vuldef + "Name")?.Value); + var product = Clean(item.Element(Vuldef + "ProductName")?.Value); + var cpeElement = item.Element(Vuldef + "Cpe"); + var cpe = Clean(cpeElement?.Value); + var cpeVendor = Clean(cpeElement?.Attribute("vendor")?.Value); + var cpeProduct = Clean(cpeElement?.Attribute("product")?.Value); + var version = Clean(ReadConcatenated(item.Elements(Vuldef + "VersionNumber"))); + var build = Clean(ReadConcatenated(item.Elements(Vuldef + "BuildNumber"))); + var description = Clean(ReadConcatenated(item.Elements(Vuldef + "Description"))); + var status = Clean(item.Attribute("affectedstatus")?.Value); + + results.Add(new JvnAffectedProductDto(vendor, product, cpe, cpeVendor, cpeProduct, version, build, description, status)); + } + + return results.ToImmutableArray(); + } + + private static ImmutableArray ParseReferences(XElement? relatedElement) + { + if (relatedElement is null) + { + return ImmutableArray.Empty; + } + + var results = new List(); + foreach (var item in relatedElement.Elements(Vuldef + "RelatedItem")) + { + var type = Clean(item.Attribute("type")?.Value) ?? string.Empty; + var id = Clean(item.Element(Vuldef + "VulinfoID")?.Value) ?? string.Empty; + var name = Clean(item.Element(Vuldef + "Name")?.Value); + var url = Clean(item.Element(Vuldef + "URL")?.Value); + + if (string.IsNullOrWhiteSpace(url)) + { + continue; + } + + if (!Uri.TryCreate(url, UriKind.Absolute, out var uri) || (uri.Scheme is not "http" and not "https")) + { + continue; + } + + results.Add(new JvnReferenceDto(type, id, name, uri.ToString())); + } + + return results.ToImmutableArray(); + } + + private static ImmutableArray ParseHistory(XElement? historyElement) + { + if (historyElement is null) + { + return ImmutableArray.Empty; + } + + var results = new List(); + foreach (var item in historyElement.Elements(Vuldef + "HistoryItem")) + { + var number = Clean(item.Element(Vuldef + "HistoryNo")?.Value); + var timestamp = ParseDate(item.Element(Vuldef + "DateTime")?.Value); + var description = Clean(item.Element(Vuldef + "Description")?.Value); + results.Add(new JvnHistoryEntryDto(number, timestamp, description)); + } + + return results.ToImmutableArray(); + } + + private static DateTimeOffset? ParseDate(string? value) + { + if (string.IsNullOrWhiteSpace(value)) + { + return null; + } + + return DateTimeOffset.TryParse(value, CultureInfo.InvariantCulture, DateTimeStyles.AssumeUniversal, out var parsed) + ? parsed.ToUniversalTime() + : null; + } + + private static string? Clean(string? value) + { + if (string.IsNullOrWhiteSpace(value)) + { + return null; + } + + return value.Trim(); + } + + private static string? ReadConcatenated(IEnumerable elements) + { + var builder = new List(); + foreach (var element in elements) + { + var text = element?.Value; + if (string.IsNullOrWhiteSpace(text)) + { + continue; + } + + builder.Add(text.Trim()); + } + + return builder.Count == 0 ? null : string.Join("; ", builder); + } +} diff --git a/src/StellaOps.Feedser.Source.Jvn/Internal/JvnOverviewItem.cs b/src/StellaOps.Feedser.Source.Jvn/Internal/JvnOverviewItem.cs new file mode 100644 index 00000000..cb421e0c --- /dev/null +++ b/src/StellaOps.Feedser.Source.Jvn/Internal/JvnOverviewItem.cs @@ -0,0 +1,8 @@ +namespace StellaOps.Feedser.Source.Jvn.Internal; + +internal sealed record JvnOverviewItem( + string VulnerabilityId, + Uri DetailUri, + string Title, + DateTimeOffset? DateFirstPublished, + DateTimeOffset? DateLastUpdated); diff --git a/src/StellaOps.Feedser.Source.Jvn/Internal/JvnOverviewPage.cs b/src/StellaOps.Feedser.Source.Jvn/Internal/JvnOverviewPage.cs new file mode 100644 index 00000000..f63779ba --- /dev/null +++ b/src/StellaOps.Feedser.Source.Jvn/Internal/JvnOverviewPage.cs @@ -0,0 +1,7 @@ +namespace StellaOps.Feedser.Source.Jvn.Internal; + +internal sealed record JvnOverviewPage( + IReadOnlyList Items, + int TotalResults, + int ReturnedCount, + int FirstResultIndex); diff --git a/src/StellaOps.Feedser.Source.Jvn/Internal/JvnSchemaProvider.cs b/src/StellaOps.Feedser.Source.Jvn/Internal/JvnSchemaProvider.cs new file mode 100644 index 00000000..075ce424 --- /dev/null +++ b/src/StellaOps.Feedser.Source.Jvn/Internal/JvnSchemaProvider.cs @@ -0,0 +1,167 @@ +using System; +using System.Collections.Generic; +using System.IO; +using System.Net; +using System.Reflection; +using System.Threading; +using System.Xml; +using System.Xml.Schema; + +namespace StellaOps.Feedser.Source.Jvn.Internal; + +internal static class JvnSchemaProvider +{ + private static readonly Lazy<(XmlSchemaSet SchemaSet, EmbeddedResourceXmlResolver Resolver)> Cached = new( + LoadSchemas, + LazyThreadSafetyMode.ExecutionAndPublication); + + public static XmlSchemaSet SchemaSet => Cached.Value.SchemaSet; + + private static (XmlSchemaSet SchemaSet, EmbeddedResourceXmlResolver Resolver) LoadSchemas() + { + var assembly = typeof(JvnSchemaProvider).GetTypeInfo().Assembly; + var resourceMap = CreateResourceMap(); + var resolver = new EmbeddedResourceXmlResolver(assembly, resourceMap); + + var schemaSet = new XmlSchemaSet + { + XmlResolver = resolver, + }; + + AddSchema(schemaSet, resolver, "https://jvndb.jvn.jp/schema/vuldef_3.2.xsd"); + AddSchema(schemaSet, resolver, "https://jvndb.jvn.jp/schema/mod_sec_3.0.xsd"); + AddSchema(schemaSet, resolver, "https://jvndb.jvn.jp/schema/status_3.3.xsd"); + AddSchema(schemaSet, resolver, "https://jvndb.jvn.jp/schema/tlp_marking.xsd"); + AddSchema(schemaSet, resolver, "https://jvndb.jvn.jp/schema/data_marking.xsd"); + + schemaSet.Compile(); + return (schemaSet, resolver); + } + + private static void AddSchema(XmlSchemaSet set, EmbeddedResourceXmlResolver resolver, string uri) + { + using var stream = resolver.OpenStream(uri); + using var reader = XmlReader.Create(stream, new XmlReaderSettings { XmlResolver = resolver }, uri); + set.Add(null, reader); + } + + private static Dictionary CreateResourceMap() + { + var baseNamespace = typeof(JvnSchemaProvider).Namespace ?? "StellaOps.Feedser.Source.Jvn.Internal"; + var prefix = baseNamespace.Replace(".Internal", string.Empty, StringComparison.Ordinal); + + return new Dictionary(StringComparer.OrdinalIgnoreCase) + { + ["https://jvndb.jvn.jp/schema/vuldef_3.2.xsd"] = $"{prefix}.Schemas.vuldef_3.2.xsd", + ["vuldef_3.2.xsd"] = $"{prefix}.Schemas.vuldef_3.2.xsd", + ["https://jvndb.jvn.jp/schema/mod_sec_3.0.xsd"] = $"{prefix}.Schemas.mod_sec_3.0.xsd", + ["mod_sec_3.0.xsd"] = $"{prefix}.Schemas.mod_sec_3.0.xsd", + ["https://jvndb.jvn.jp/schema/status_3.3.xsd"] = $"{prefix}.Schemas.status_3.3.xsd", + ["status_3.3.xsd"] = $"{prefix}.Schemas.status_3.3.xsd", + ["https://jvndb.jvn.jp/schema/tlp_marking.xsd"] = $"{prefix}.Schemas.tlp_marking.xsd", + ["tlp_marking.xsd"] = $"{prefix}.Schemas.tlp_marking.xsd", + ["https://jvndb.jvn.jp/schema/data_marking.xsd"] = $"{prefix}.Schemas.data_marking.xsd", + ["data_marking.xsd"] = $"{prefix}.Schemas.data_marking.xsd", + ["https://www.w3.org/2001/xml.xsd"] = $"{prefix}.Schemas.xml.xsd", + ["xml.xsd"] = $"{prefix}.Schemas.xml.xsd", + }; + } + + private sealed class EmbeddedResourceXmlResolver : XmlResolver + { + private readonly Assembly _assembly; + private readonly Dictionary _resourceMap; + + public EmbeddedResourceXmlResolver(Assembly assembly, Dictionary resourceMap) + { + _assembly = assembly ?? throw new ArgumentNullException(nameof(assembly)); + _resourceMap = resourceMap ?? throw new ArgumentNullException(nameof(resourceMap)); + } + + public override ICredentials? Credentials + { + set { } + } + + public Stream OpenStream(string uriOrName) + { + var resourceName = ResolveResourceName(uriOrName) + ?? throw new FileNotFoundException($"Schema resource '{uriOrName}' not found in manifest."); + + var stream = _assembly.GetManifestResourceStream(resourceName); + if (stream is null) + { + throw new FileNotFoundException($"Embedded schema '{resourceName}' could not be opened."); + } + + return stream; + } + + public override object? GetEntity(Uri absoluteUri, string? role, Type? ofObjectToReturn) + { + if (absoluteUri is null) + { + throw new ArgumentNullException(nameof(absoluteUri)); + } + + var resourceName = ResolveResourceName(absoluteUri.AbsoluteUri) + ?? ResolveResourceName(absoluteUri.AbsolutePath.TrimStart('/')) + ?? ResolveResourceName(Path.GetFileName(absoluteUri.AbsolutePath)) + ?? throw new FileNotFoundException($"Schema resource for '{absoluteUri}' not found."); + + var stream = _assembly.GetManifestResourceStream(resourceName); + if (stream is null) + { + throw new FileNotFoundException($"Embedded schema '{resourceName}' could not be opened."); + } + + return stream; + } + + public override Uri ResolveUri(Uri? baseUri, string? relativeUri) + { + if (string.IsNullOrWhiteSpace(relativeUri)) + { + return base.ResolveUri(baseUri, relativeUri); + } + + if (Uri.TryCreate(relativeUri, UriKind.Absolute, out var absolute)) + { + return absolute; + } + + if (baseUri is not null && Uri.TryCreate(baseUri, relativeUri, out var combined)) + { + return combined; + } + + if (_resourceMap.ContainsKey(relativeUri)) + { + return new Uri($"embedded:///{relativeUri}", UriKind.Absolute); + } + + return base.ResolveUri(baseUri, relativeUri); + } + + private string? ResolveResourceName(string? key) + { + if (string.IsNullOrWhiteSpace(key)) + { + return null; + } + + if (_resourceMap.TryGetValue(key, out var resource)) + { + return resource; + } + + var fileName = Path.GetFileName(key); + if (!string.IsNullOrEmpty(fileName) && _resourceMap.TryGetValue(fileName, out resource)) + { + return resource; + } + + return null; + } + } +} diff --git a/src/StellaOps.Feedser.Source.Jvn/Internal/JvnSchemaValidationException.cs b/src/StellaOps.Feedser.Source.Jvn/Internal/JvnSchemaValidationException.cs new file mode 100644 index 00000000..0015fa5f --- /dev/null +++ b/src/StellaOps.Feedser.Source.Jvn/Internal/JvnSchemaValidationException.cs @@ -0,0 +1,16 @@ +using System; + +namespace StellaOps.Feedser.Source.Jvn.Internal; + +internal sealed class JvnSchemaValidationException : Exception +{ + public JvnSchemaValidationException(string message) + : base(message) + { + } + + public JvnSchemaValidationException(string message, Exception innerException) + : base(message, innerException) + { + } +} diff --git a/src/StellaOps.Feedser.Source.Jvn/Internal/MyJvnClient.cs b/src/StellaOps.Feedser.Source.Jvn/Internal/MyJvnClient.cs new file mode 100644 index 00000000..319358cc --- /dev/null +++ b/src/StellaOps.Feedser.Source.Jvn/Internal/MyJvnClient.cs @@ -0,0 +1,240 @@ +using System.Collections.Generic; +using System.Globalization; +using System.Linq; +using System.Net; +using System.Net.Http; +using System.Threading; +using System.Threading.Tasks; +using System.Xml; +using System.Xml.Linq; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Options; +using StellaOps.Feedser.Source.Jvn.Configuration; + +namespace StellaOps.Feedser.Source.Jvn.Internal; + +public sealed class MyJvnClient +{ + private static readonly XNamespace RssNamespace = "http://purl.org/rss/1.0/"; + private static readonly XNamespace DcTermsNamespace = "http://purl.org/dc/terms/"; + private static readonly XNamespace SecNamespace = "http://jvn.jp/rss/mod_sec/3.0/"; + private static readonly XNamespace RdfNamespace = "http://www.w3.org/1999/02/22-rdf-syntax-ns#"; + private static readonly XNamespace StatusNamespace = "http://jvndb.jvn.jp/myjvn/Status"; + + private static readonly TimeSpan TokyoOffset = TimeSpan.FromHours(9); + + private readonly IHttpClientFactory _httpClientFactory; + private readonly JvnOptions _options; + private readonly ILogger _logger; + + public MyJvnClient(IHttpClientFactory httpClientFactory, IOptions options, ILogger logger) + { + _httpClientFactory = httpClientFactory ?? throw new ArgumentNullException(nameof(httpClientFactory)); + _options = (options ?? throw new ArgumentNullException(nameof(options))).Value ?? throw new ArgumentNullException(nameof(options)); + _options.Validate(); + _logger = logger ?? throw new ArgumentNullException(nameof(logger)); + } + + internal async Task> GetOverviewAsync(DateTimeOffset windowStart, DateTimeOffset windowEnd, CancellationToken cancellationToken) + { + if (windowEnd <= windowStart) + { + throw new ArgumentException("windowEnd must be greater than windowStart", nameof(windowEnd)); + } + + var items = new List(); + var client = _httpClientFactory.CreateClient(JvnOptions.HttpClientName); + + var startItem = 1; + var pagesFetched = 0; + + while (pagesFetched < _options.MaxOverviewPagesPerFetch) + { + cancellationToken.ThrowIfCancellationRequested(); + + var requestUri = BuildOverviewUri(windowStart, windowEnd, startItem); + using var response = await client.GetAsync(requestUri, cancellationToken).ConfigureAwait(false); + response.EnsureSuccessStatusCode(); + + await using var contentStream = await response.Content.ReadAsStreamAsync(cancellationToken).ConfigureAwait(false); + using var reader = XmlReader.Create(contentStream, new XmlReaderSettings { Async = true, IgnoreWhitespace = true, IgnoreComments = true }); + var document = await XDocument.LoadAsync(reader, LoadOptions.None, cancellationToken).ConfigureAwait(false); + + var page = ParseOverviewPage(document); + if (page.Items.Count == 0) + { + _logger.LogDebug("JVN overview page starting at {StartItem} returned zero results", startItem); + break; + } + + items.AddRange(page.Items); + pagesFetched++; + + if (page.ReturnedCount < _options.PageSize || startItem + _options.PageSize > page.TotalResults) + { + break; + } + + startItem += _options.PageSize; + + if (_options.RequestDelay > TimeSpan.Zero) + { + try + { + await Task.Delay(_options.RequestDelay, cancellationToken).ConfigureAwait(false); + } + catch (TaskCanceledException) + { + break; + } + } + } + + return items; + } + + private Uri BuildOverviewUri(DateTimeOffset windowStart, DateTimeOffset windowEnd, int startItem) + { + var (startYear, startMonth, startDay) = ToTokyoDateParts(windowStart); + var (endYear, endMonth, endDay) = ToTokyoDateParts(windowEnd); + + var parameters = new[] + { + new KeyValuePair("method", "getVulnOverviewList"), + new KeyValuePair("feed", "hnd"), + new KeyValuePair("lang", "en"), + new KeyValuePair("rangeDatePublished", "n"), + new KeyValuePair("rangeDatePublic", "n"), + new KeyValuePair("rangeDateFirstPublished", "n"), + new KeyValuePair("dateFirstPublishedStartY", startYear), + new KeyValuePair("dateFirstPublishedStartM", startMonth), + new KeyValuePair("dateFirstPublishedStartD", startDay), + new KeyValuePair("dateFirstPublishedEndY", endYear), + new KeyValuePair("dateFirstPublishedEndM", endMonth), + new KeyValuePair("dateFirstPublishedEndD", endDay), + new KeyValuePair("startItem", startItem.ToString(CultureInfo.InvariantCulture)), + new KeyValuePair("maxCountItem", _options.PageSize.ToString(CultureInfo.InvariantCulture)), + }; + + var query = BuildQueryString(parameters); + + var builder = new UriBuilder(_options.BaseEndpoint) + { + Query = query, + }; + return builder.Uri; + } + + private static (string Year, string Month, string Day) ToTokyoDateParts(DateTimeOffset timestamp) + { + var local = timestamp.ToOffset(TokyoOffset).Date; + return ( + local.Year.ToString("D4", CultureInfo.InvariantCulture), + local.Month.ToString("D2", CultureInfo.InvariantCulture), + local.Day.ToString("D2", CultureInfo.InvariantCulture)); + } + + private static JvnOverviewPage ParseOverviewPage(XDocument document) + { + var items = new List(); + + foreach (var item in document.Descendants(RssNamespace + "item")) + { + var identifier = item.Element(SecNamespace + "identifier")?.Value?.Trim(); + if (string.IsNullOrWhiteSpace(identifier)) + { + continue; + } + + Uri? detailUri = null; + var linkValue = item.Element(RssNamespace + "link")?.Value?.Trim(); + if (!string.IsNullOrWhiteSpace(linkValue)) + { + Uri.TryCreate(linkValue, UriKind.Absolute, out detailUri); + } + + if (detailUri is null) + { + var aboutValue = item.Attribute(RdfNamespace + "about")?.Value?.Trim(); + if (!string.IsNullOrWhiteSpace(aboutValue)) + { + Uri.TryCreate(aboutValue, UriKind.Absolute, out detailUri); + } + } + + if (detailUri is null) + { + continue; + } + + var title = item.Element(RssNamespace + "title")?.Value?.Trim(); + if (string.IsNullOrWhiteSpace(title)) + { + title = identifier; + } + + var firstPublished = TryParseDate(item.Element(DcTermsNamespace + "issued")?.Value); + var lastUpdated = TryParseDate(item.Element(DcTermsNamespace + "modified")?.Value); + + items.Add(new JvnOverviewItem(identifier, detailUri, title!, firstPublished, lastUpdated)); + } + + var statusElement = document.Root?.Element(StatusNamespace + "Status") + ?? document.Descendants(StatusNamespace + "Status").FirstOrDefault(); + + var totalResults = TryParseInt(statusElement?.Attribute("totalRes")?.Value) ?? items.Count; + var returned = TryParseInt(statusElement?.Attribute("totalResRet")?.Value) ?? items.Count; + var firstResult = TryParseInt(statusElement?.Attribute("firstRes")?.Value) ?? 1; + + return new JvnOverviewPage(items, totalResults, returned, firstResult); + } + + private static DateTimeOffset? TryParseDate(string? value) + { + if (string.IsNullOrWhiteSpace(value)) + { + return null; + } + + return DateTimeOffset.TryParse(value, CultureInfo.InvariantCulture, DateTimeStyles.AdjustToUniversal, out var parsed) + ? parsed.ToUniversalTime() + : null; + } + + private static int? TryParseInt(string? value) + { + if (string.IsNullOrWhiteSpace(value)) + { + return null; + } + + return int.TryParse(value, NumberStyles.Integer, CultureInfo.InvariantCulture, out var parsed) ? parsed : null; + } + + internal Uri BuildDetailUri(string vulnerabilityId) + { + ArgumentException.ThrowIfNullOrEmpty(vulnerabilityId); + + var query = BuildQueryString(new[] + { + new KeyValuePair("method", "getVulnDetailInfo"), + new KeyValuePair("feed", "hnd"), + new KeyValuePair("lang", "en"), + new KeyValuePair("vulnId", vulnerabilityId.Trim()), + }); + var builder = new UriBuilder(_options.BaseEndpoint) + { + Query = query, + }; + + return builder.Uri; + } + + private static string BuildQueryString(IEnumerable> parameters) + { + return string.Join( + "&", + parameters.Select(parameter => + $"{WebUtility.UrlEncode(parameter.Key)}={WebUtility.UrlEncode(parameter.Value)}")); + } +} diff --git a/src/StellaOps.Feedser.Source.Jvn/Jobs.cs b/src/StellaOps.Feedser.Source.Jvn/Jobs.cs new file mode 100644 index 00000000..e56571d1 --- /dev/null +++ b/src/StellaOps.Feedser.Source.Jvn/Jobs.cs @@ -0,0 +1,46 @@ +using System; +using System.Threading; +using System.Threading.Tasks; +using StellaOps.Feedser.Core.Jobs; + +namespace StellaOps.Feedser.Source.Jvn; + +internal static class JvnJobKinds +{ + public const string Fetch = "source:jvn:fetch"; + public const string Parse = "source:jvn:parse"; + public const string Map = "source:jvn:map"; +} + +internal sealed class JvnFetchJob : IJob +{ + private readonly JvnConnector _connector; + + public JvnFetchJob(JvnConnector connector) + => _connector = connector ?? throw new ArgumentNullException(nameof(connector)); + + public Task ExecuteAsync(JobExecutionContext context, CancellationToken cancellationToken) + => _connector.FetchAsync(context.Services, cancellationToken); +} + +internal sealed class JvnParseJob : IJob +{ + private readonly JvnConnector _connector; + + public JvnParseJob(JvnConnector connector) + => _connector = connector ?? throw new ArgumentNullException(nameof(connector)); + + public Task ExecuteAsync(JobExecutionContext context, CancellationToken cancellationToken) + => _connector.ParseAsync(context.Services, cancellationToken); +} + +internal sealed class JvnMapJob : IJob +{ + private readonly JvnConnector _connector; + + public JvnMapJob(JvnConnector connector) + => _connector = connector ?? throw new ArgumentNullException(nameof(connector)); + + public Task ExecuteAsync(JobExecutionContext context, CancellationToken cancellationToken) + => _connector.MapAsync(context.Services, cancellationToken); +} diff --git a/src/StellaOps.Feedser.Source.Jvn/JvnConnector.cs b/src/StellaOps.Feedser.Source.Jvn/JvnConnector.cs new file mode 100644 index 00000000..0be9aadf --- /dev/null +++ b/src/StellaOps.Feedser.Source.Jvn/JvnConnector.cs @@ -0,0 +1,325 @@ +using System.Collections.Generic; +using System.Linq; +using System.Text.Json; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Options; +using MongoDB.Bson; +using StellaOps.Feedser.Models; +using StellaOps.Feedser.Source.Common; +using StellaOps.Feedser.Source.Common.Fetch; +using StellaOps.Feedser.Source.Jvn.Configuration; +using StellaOps.Feedser.Source.Jvn.Internal; +using StellaOps.Feedser.Storage.Mongo; +using StellaOps.Feedser.Storage.Mongo.Advisories; +using StellaOps.Feedser.Storage.Mongo.Documents; +using StellaOps.Feedser.Storage.Mongo.Dtos; +using StellaOps.Feedser.Storage.Mongo.JpFlags; +using StellaOps.Plugin; + +namespace StellaOps.Feedser.Source.Jvn; + +public sealed class JvnConnector : IFeedConnector +{ + private static readonly JsonSerializerOptions SerializerOptions = new(JsonSerializerDefaults.General) + { + PropertyNamingPolicy = JsonNamingPolicy.CamelCase, + WriteIndented = false, + DefaultIgnoreCondition = System.Text.Json.Serialization.JsonIgnoreCondition.WhenWritingNull, + }; + + private readonly MyJvnClient _client; + private readonly SourceFetchService _fetchService; + private readonly RawDocumentStorage _rawDocumentStorage; + private readonly IDocumentStore _documentStore; + private readonly IDtoStore _dtoStore; + private readonly IAdvisoryStore _advisoryStore; + private readonly IJpFlagStore _jpFlagStore; + private readonly ISourceStateRepository _stateRepository; + private readonly TimeProvider _timeProvider; + private readonly JvnOptions _options; + private readonly ILogger _logger; + + public JvnConnector( + MyJvnClient client, + SourceFetchService fetchService, + RawDocumentStorage rawDocumentStorage, + IDocumentStore documentStore, + IDtoStore dtoStore, + IAdvisoryStore advisoryStore, + IJpFlagStore jpFlagStore, + ISourceStateRepository stateRepository, + IOptions options, + TimeProvider? timeProvider, + ILogger logger) + { + _client = client ?? throw new ArgumentNullException(nameof(client)); + _fetchService = fetchService ?? throw new ArgumentNullException(nameof(fetchService)); + _rawDocumentStorage = rawDocumentStorage ?? throw new ArgumentNullException(nameof(rawDocumentStorage)); + _documentStore = documentStore ?? throw new ArgumentNullException(nameof(documentStore)); + _dtoStore = dtoStore ?? throw new ArgumentNullException(nameof(dtoStore)); + _advisoryStore = advisoryStore ?? throw new ArgumentNullException(nameof(advisoryStore)); + _jpFlagStore = jpFlagStore ?? throw new ArgumentNullException(nameof(jpFlagStore)); + _stateRepository = stateRepository ?? throw new ArgumentNullException(nameof(stateRepository)); + _options = (options ?? throw new ArgumentNullException(nameof(options))).Value ?? throw new ArgumentNullException(nameof(options)); + _options.Validate(); + _timeProvider = timeProvider ?? TimeProvider.System; + _logger = logger ?? throw new ArgumentNullException(nameof(logger)); + } + + public string SourceName => JvnConnectorPlugin.SourceName; + + public async Task FetchAsync(IServiceProvider services, CancellationToken cancellationToken) + { + ArgumentNullException.ThrowIfNull(services); + + var cursor = await GetCursorAsync(cancellationToken).ConfigureAwait(false); + var now = _timeProvider.GetUtcNow(); + + var windowEnd = now; + var defaultWindowStart = windowEnd - _options.WindowSize; + + var windowStart = cursor.LastCompletedWindowEnd.HasValue + ? cursor.LastCompletedWindowEnd.Value - _options.WindowOverlap + : defaultWindowStart; + + if (windowStart < defaultWindowStart) + { + windowStart = defaultWindowStart; + } + + if (windowStart >= windowEnd) + { + windowStart = windowEnd - TimeSpan.FromHours(1); + } + + _logger.LogInformation("JVN fetch window {WindowStart:o} - {WindowEnd:o}", windowStart, windowEnd); + + IReadOnlyList overviewItems; + try + { + overviewItems = await _client.GetOverviewAsync(windowStart, windowEnd, cancellationToken).ConfigureAwait(false); + } + catch (Exception ex) + { + _logger.LogError(ex, "Failed to retrieve JVN overview between {Start:o} and {End:o}", windowStart, windowEnd); + await _stateRepository.MarkFailureAsync(SourceName, now, TimeSpan.FromMinutes(5), ex.Message, cancellationToken).ConfigureAwait(false); + throw; + } + + _logger.LogInformation("JVN overview returned {Count} items", overviewItems.Count); + + var pendingDocuments = cursor.PendingDocuments.ToHashSet(); + + foreach (var item in overviewItems) + { + cancellationToken.ThrowIfCancellationRequested(); + + var detailUri = _client.BuildDetailUri(item.VulnerabilityId); + var metadata = new Dictionary(StringComparer.Ordinal) + { + ["jvn.vulnId"] = item.VulnerabilityId, + ["jvn.detailUrl"] = detailUri.ToString(), + }; + + if (item.DateFirstPublished.HasValue) + { + metadata["jvn.firstPublished"] = item.DateFirstPublished.Value.ToString("O"); + } + + if (item.DateLastUpdated.HasValue) + { + metadata["jvn.lastUpdated"] = item.DateLastUpdated.Value.ToString("O"); + } + + var result = await _fetchService.FetchAsync( + new SourceFetchRequest(JvnOptions.HttpClientName, SourceName, detailUri) + { + Metadata = metadata + }, + cancellationToken).ConfigureAwait(false); + + if (!result.IsSuccess || result.Document is null) + { + if (!result.IsNotModified) + { + _logger.LogWarning("JVN fetch for {Uri} returned status {Status}", detailUri, result.StatusCode); + } + + continue; + } + + _logger.LogDebug("JVN fetched document {DocumentId}", result.Document.Id); + pendingDocuments.Add(result.Document.Id); + } + + var updatedCursor = cursor + .WithWindow(windowStart, windowEnd) + .WithCompletedWindow(windowEnd) + .WithPendingDocuments(pendingDocuments); + + await UpdateCursorAsync(updatedCursor, cancellationToken).ConfigureAwait(false); + } + + public async Task ParseAsync(IServiceProvider services, CancellationToken cancellationToken) + { + ArgumentNullException.ThrowIfNull(services); + + var cursor = await GetCursorAsync(cancellationToken).ConfigureAwait(false); + _logger.LogDebug("JVN parse pending documents: {PendingCount}", cursor.PendingDocuments.Count); + Console.WriteLine($"JVN parse pending count: {cursor.PendingDocuments.Count}"); + if (cursor.PendingDocuments.Count == 0) + { + return; + } + + var remainingDocuments = cursor.PendingDocuments.ToList(); + var pendingMappings = cursor.PendingMappings.ToList(); + + foreach (var documentId in cursor.PendingDocuments) + { + cancellationToken.ThrowIfCancellationRequested(); + _logger.LogDebug("JVN parsing document {DocumentId}", documentId); + Console.WriteLine($"JVN parsing document {documentId}"); + + var document = await _documentStore.FindAsync(documentId, cancellationToken).ConfigureAwait(false); + if (document is null) + { + _logger.LogWarning("JVN document {DocumentId} no longer exists; skipping", documentId); + remainingDocuments.Remove(documentId); + continue; + } + + if (!document.GridFsId.HasValue) + { + _logger.LogWarning("JVN document {DocumentId} is missing GridFS content; marking as failed", documentId); + await _documentStore.UpdateStatusAsync(document.Id, DocumentStatuses.Failed, cancellationToken).ConfigureAwait(false); + remainingDocuments.Remove(documentId); + continue; + } + + byte[] rawBytes; + try + { + rawBytes = await _rawDocumentStorage.DownloadAsync(document.GridFsId.Value, cancellationToken).ConfigureAwait(false); + } + catch (Exception ex) + { + _logger.LogError(ex, "Unable to download raw JVN document {DocumentId}", document.Id); + throw; + } + + JvnDetailDto detail; + try + { + detail = JvnDetailParser.Parse(rawBytes, document.Uri); + } + catch (JvnSchemaValidationException ex) + { + Console.WriteLine($"JVN schema validation exception: {ex.Message}"); + _logger.LogWarning(ex, "JVN schema validation failed for document {DocumentId} ({Uri})", document.Id, document.Uri); + await _documentStore.UpdateStatusAsync(document.Id, DocumentStatuses.Failed, cancellationToken).ConfigureAwait(false); + remainingDocuments.Remove(documentId); + throw; + } + + var sanitizedJson = JsonSerializer.Serialize(detail, SerializerOptions); + var payload = BsonDocument.Parse(sanitizedJson); + var dtoRecord = new DtoRecord( + Guid.NewGuid(), + document.Id, + SourceName, + JvnConstants.DtoSchemaVersion, + payload, + _timeProvider.GetUtcNow()); + + await _dtoStore.UpsertAsync(dtoRecord, cancellationToken).ConfigureAwait(false); + await _documentStore.UpdateStatusAsync(document.Id, DocumentStatuses.PendingMap, cancellationToken).ConfigureAwait(false); + + remainingDocuments.Remove(documentId); + if (!pendingMappings.Contains(documentId)) + { + pendingMappings.Add(documentId); + Console.WriteLine($"Added mapping for {documentId}"); + _logger.LogDebug("JVN parsed document {DocumentId}", documentId); + } + } + + var updatedCursor = cursor + .WithPendingDocuments(remainingDocuments) + .WithPendingMappings(pendingMappings); + + await UpdateCursorAsync(updatedCursor, cancellationToken).ConfigureAwait(false); + } + + public async Task MapAsync(IServiceProvider services, CancellationToken cancellationToken) + { + ArgumentNullException.ThrowIfNull(services); + + var cursor = await GetCursorAsync(cancellationToken).ConfigureAwait(false); + _logger.LogDebug("JVN map pending mappings: {PendingCount}", cursor.PendingMappings.Count); + if (cursor.PendingMappings.Count == 0) + { + return; + } + + var pendingMappings = cursor.PendingMappings.ToList(); + + foreach (var documentId in cursor.PendingMappings) + { + cancellationToken.ThrowIfCancellationRequested(); + + var dto = await _dtoStore.FindByDocumentIdAsync(documentId, cancellationToken).ConfigureAwait(false); + var document = await _documentStore.FindAsync(documentId, cancellationToken).ConfigureAwait(false); + + if (dto is null || document is null) + { + _logger.LogWarning("Skipping JVN mapping for {DocumentId}: DTO or document missing", documentId); + pendingMappings.Remove(documentId); + continue; + } + + var dtoJson = dto.Payload.ToJson(new MongoDB.Bson.IO.JsonWriterSettings + { + OutputMode = MongoDB.Bson.IO.JsonOutputMode.RelaxedExtendedJson, + }); + + JvnDetailDto detail; + try + { + detail = JsonSerializer.Deserialize(dtoJson, SerializerOptions) + ?? throw new InvalidOperationException("Deserialized DTO was null."); + } + catch (Exception ex) + { + _logger.LogError(ex, "Failed to deserialize JVN DTO for document {DocumentId}", document.Id); + await _documentStore.UpdateStatusAsync(document.Id, DocumentStatuses.Failed, cancellationToken).ConfigureAwait(false); + pendingMappings.Remove(documentId); + continue; + } + + var (advisory, flag) = JvnAdvisoryMapper.Map(detail, document, dto, _timeProvider); + + await _advisoryStore.UpsertAsync(advisory, cancellationToken).ConfigureAwait(false); + await _jpFlagStore.UpsertAsync(flag, cancellationToken).ConfigureAwait(false); + await _documentStore.UpdateStatusAsync(document.Id, DocumentStatuses.Mapped, cancellationToken).ConfigureAwait(false); + + pendingMappings.Remove(documentId); + _logger.LogDebug("JVN mapped document {DocumentId}", documentId); + } + + var updatedCursor = cursor.WithPendingMappings(pendingMappings); + await UpdateCursorAsync(updatedCursor, cancellationToken).ConfigureAwait(false); + } + + private async Task GetCursorAsync(CancellationToken cancellationToken) + { + var state = await _stateRepository.TryGetAsync(SourceName, cancellationToken).ConfigureAwait(false); + return state is null ? JvnCursor.Empty : JvnCursor.FromBson(state.Cursor); + } + + private async Task UpdateCursorAsync(JvnCursor cursor, CancellationToken cancellationToken) + { + var cursorDocument = cursor.ToBsonDocument(); + await _stateRepository.UpdateCursorAsync(SourceName, cursorDocument, _timeProvider.GetUtcNow(), cancellationToken).ConfigureAwait(false); + } +} diff --git a/src/StellaOps.Feedser.Source.Jvn/JvnConnectorPlugin.cs b/src/StellaOps.Feedser.Source.Jvn/JvnConnectorPlugin.cs new file mode 100644 index 00000000..406c4cb2 --- /dev/null +++ b/src/StellaOps.Feedser.Source.Jvn/JvnConnectorPlugin.cs @@ -0,0 +1,19 @@ +using Microsoft.Extensions.DependencyInjection; +using StellaOps.Plugin; + +namespace StellaOps.Feedser.Source.Jvn; + +public sealed class JvnConnectorPlugin : IConnectorPlugin +{ + public string Name => SourceName; + + public static string SourceName => "jvn"; + + public bool IsAvailable(IServiceProvider services) => services is not null; + + public IFeedConnector Create(IServiceProvider services) + { + ArgumentNullException.ThrowIfNull(services); + return ActivatorUtilities.CreateInstance(services); + } +} diff --git a/src/StellaOps.Feedser.Source.Jvn/JvnDependencyInjectionRoutine.cs b/src/StellaOps.Feedser.Source.Jvn/JvnDependencyInjectionRoutine.cs new file mode 100644 index 00000000..1b7accd1 --- /dev/null +++ b/src/StellaOps.Feedser.Source.Jvn/JvnDependencyInjectionRoutine.cs @@ -0,0 +1,54 @@ +using System; +using Microsoft.Extensions.Configuration; +using Microsoft.Extensions.DependencyInjection; +using StellaOps.DependencyInjection; +using StellaOps.Feedser.Core.Jobs; +using StellaOps.Feedser.Source.Jvn.Configuration; + +namespace StellaOps.Feedser.Source.Jvn; + +public sealed class JvnDependencyInjectionRoutine : IDependencyInjectionRoutine +{ + private const string ConfigurationSection = "feedser:sources:jvn"; + + public IServiceCollection Register(IServiceCollection services, IConfiguration configuration) + { + ArgumentNullException.ThrowIfNull(services); + ArgumentNullException.ThrowIfNull(configuration); + + services.AddJvnConnector(options => + { + configuration.GetSection(ConfigurationSection).Bind(options); + options.Validate(); + }); + + services.AddTransient(); + services.AddTransient(); + services.AddTransient(); + + services.PostConfigure(options => + { + EnsureJob(options, JvnJobKinds.Fetch, typeof(JvnFetchJob)); + EnsureJob(options, JvnJobKinds.Parse, typeof(JvnParseJob)); + EnsureJob(options, JvnJobKinds.Map, typeof(JvnMapJob)); + }); + + return services; + } + + private static void EnsureJob(JobSchedulerOptions options, string kind, Type jobType) + { + if (options.Definitions.ContainsKey(kind)) + { + return; + } + + options.Definitions[kind] = new JobDefinition( + kind, + jobType, + options.DefaultTimeout, + options.DefaultLeaseDuration, + CronExpression: null, + Enabled: true); + } +} diff --git a/src/StellaOps.Feedser.Source.Jvn/JvnServiceCollectionExtensions.cs b/src/StellaOps.Feedser.Source.Jvn/JvnServiceCollectionExtensions.cs new file mode 100644 index 00000000..2ce8d662 --- /dev/null +++ b/src/StellaOps.Feedser.Source.Jvn/JvnServiceCollectionExtensions.cs @@ -0,0 +1,37 @@ +using System; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Options; +using StellaOps.Feedser.Source.Common.Http; +using StellaOps.Feedser.Source.Jvn.Configuration; +using StellaOps.Feedser.Source.Jvn.Internal; + +namespace StellaOps.Feedser.Source.Jvn; + +public static class JvnServiceCollectionExtensions +{ + public static IServiceCollection AddJvnConnector(this IServiceCollection services, Action configure) + { + ArgumentNullException.ThrowIfNull(services); + ArgumentNullException.ThrowIfNull(configure); + + services.AddOptions() + .Configure(configure) + .PostConfigure(static options => options.Validate()); + + services.AddSourceHttpClient(JvnOptions.HttpClientName, (sp, clientOptions) => + { + var options = sp.GetRequiredService>().Value; + clientOptions.BaseAddress = options.BaseEndpoint; + clientOptions.Timeout = TimeSpan.FromSeconds(30); + clientOptions.UserAgent = "StellaOps.Feedser.Jvn/1.0"; + clientOptions.AllowedHosts.Clear(); + clientOptions.AllowedHosts.Add(options.BaseEndpoint.Host); + clientOptions.DefaultRequestHeaders["Accept"] = "application/xml"; + }); + + services.AddTransient(); + services.AddTransient(); + + return services; + } +} diff --git a/src/StellaOps.Feedser.Source.Jvn/Schemas/data_marking.xsd b/src/StellaOps.Feedser.Source.Jvn/Schemas/data_marking.xsd new file mode 100644 index 00000000..7d077410 --- /dev/null +++ b/src/StellaOps.Feedser.Source.Jvn/Schemas/data_marking.xsd @@ -0,0 +1,91 @@ + + + + This schema was originally developed by The MITRE Corporation. The Data Marking XML Schema implementation is maintained by The MITRE Corporation and developed by the open STIX Community. For more information, including how to get involved in the effort and how to submit change requests, please visit the STIX website at http://stix.mitre.org. + + Data Marking + 1.1.1 + 05/08/2014 9:00:00 AM + Data_Marking - Schematic implementation for an independent, flexible, structured data marking expression. + Copyright (c) 2012-2014, The MITRE Corporation. All rights reserved. The contents of this file are subject to the terms of the STIX License located at http://stix.mitre.org/about/termsofuse.html. See the STIX License for the specific language governing permissions and limitations for use of this schema. When distributing copies of the Data Marking Schema, this license header must be included. + + + + + MarkingType specifies a structure for marking information to be applied to portions of XML content. + + + + + This field contains specification of marking information to be applied to portions of XML content. + + + + + + + The MarkingStructureType contains the marking information to be applied to a portion of XML content. + This type is defined as abstract and is intended to be extended to enable the expression of any structured or unstructured data marking mechanism. The data marking structure is simply a mechanism for applying existing marking systems to nodes. The data marking systems themselves define the semantics of what the markings mean, how multiple markings to the same node should be applied, and what to do if a node is unmarked. + It is valid per this specification to mark a node with multiple markings from the same system or mark a node across multiple marking systems. If a node is marked multiple times using the same marking system, that system specifies the semantic meaning of multiple markings and (if necessary) how conflicts should be resolved. If a node is marked across multiple marking systems, each system is considered individually applicable. If there are conflicting markings across marking systems the behavior is undefined, therefore producers should make every effort to ensure documents are marked consistently and correctly among all marking systems. + STIX provides two marking system extensions: Simple, and TLP. Those who wish to use another format may do so by defining a new extension to this type. The STIX-provided extensions are: + 1. Simple: The Simple marking structure allows for the specification of unstructured statements through the use of a string field. The type is named SimpleMarkingStructureType and is in the http://data-marking.mitre.org/extensions/MarkingStructure#Simple-1 namespace. The extension is defined in the file extensions/marking/simple_marking.xsd or at the URL http://stix.mitre.org/XMLSchema/extensions/marking/simple_marking/1.1.1/simple_marking.xsd. + 2. TLP: The TLP marking structure allows for the expression of Traffic Light Protocol statements through the use of a simple enumeration. The type is named TLPMarkingStructureType and is in the http://data-marking.mitre.org/extensions/MarkingStructure#TLP-1 namespace. The extension is defined in the file extensions/marking/tlp_marking.xsd or at the URL http://stix.mitre.org/XMLSchema/extensions/marking/tlp/1.1.1/tlp_marking.xsd. + 3. Terms of Use: The Terms of Use marking structure allows for the specification of unstructured terms of use statements through the use of a string field. The type is named TermsOfUseMarkingStructureType and is in the http://data-marking.mitre.org/extensions/MarkingStructure#Terms_Of_Use-1 namespace. The extension is defined in the file extensions/marking/terms_of_use_marking.xsd or at the URL http://stix.mitre.org/XMLSchema/extensions/marking/terms_of_use/1.0.1/terms_of_use_marking.xsd. + + + + This field specifies the name of the marking model to be applied within this Marking_Structure. + + + + + This field contains a reference to an authoritative source on the marking model to be applied within this Marking_Structure. + + + + + Specifies a unique ID for this Marking_Structure. + + + + + Specifies a reference to the ID of a Marking_Structure defined elsewhere. + When idref is specified, the id attribute must not be specified, and any instance of this Marking_Structure should not hold content. + + + + + + + + This field utilizes XPath 1.0 to specify the structures for which the Marking is to be applied. + The XPath expression is NOT recursive and the marking structure does NOT apply to child nodes of the selected node. Instead, the expression must explicitly select all nodes that the marking is to be applied to including elements, attributes, and text nodes. + The context root of the XPath statement is this Controlled_Structure element. Any namespace prefix declarations that are available to this Controlled_Structure element are available to the XPath. + Note that all Controlled_Structure elements have a scope within the document for which their XPath is valid to reference. + Usages of MarkingType may specify a "marking scope". The marking scope is always recursive and specifies the set of nodes that may be selected by the XPath expression (and therefore that may have the markings applied to them). If no marking scope is specified in the schema documentation or specification where the MarkingType is used, it should be assumed that the document itself and all nodes are within scope. + + + + + This field contains the marking information to be applied to the portions of XML content specified in the ControlledStructure field. This field is defined as MarkingStructureType which is an abstract type the enables the flexibility to utilize any variety of marking structures. + + + + + + Specifies a unique ID for this Marking. + + + + + Specifies a reference to the ID of a Marking defined elsewhere. + When idref is specified, the id attribute must not be specified, and any instance of this Marking should not hold content. + + + + + Specifies the relevant Data_Marking schema version for this content. + + + + diff --git a/src/StellaOps.Feedser.Source.Jvn/Schemas/jvnrss_3.2.xsd b/src/StellaOps.Feedser.Source.Jvn/Schemas/jvnrss_3.2.xsd new file mode 100644 index 00000000..66599ff8 --- /dev/null +++ b/src/StellaOps.Feedser.Source.Jvn/Schemas/jvnrss_3.2.xsd @@ -0,0 +1,133 @@ + + + + + + + + + + + + + + + + + + + JVNRSS is based on RDF Site Summary (RSS) 1.0 and use the + field dc:relation of Dublin Core / sec:references of mod_sec as index of grouping + security information. + JVNRSS は、脆弱性対策情報の概要記述用 XML フォーマットで、サイトの概要をメタデータとして簡潔に記述する + XML フォーマットである RSS (RDF Site Summary) 1.0 をベースとした仕様です。他サイトに掲載可能な形式で発信する仕組み、脆弱性対策情報のグループ化 + (dc:relation, sec:references) + や抽出した情報の再構成などの点から、脆弱性対策情報の利活用を促進することを目的としています。 + https://jvndb.jvn.jp/en/schema/jvnrss.html + https://jvndb.jvn.jp/schema/jvnrss.html + + JVN RDF Site Summary (JVNRSS) + Masato Terada + 3.2 + 2017-07-20T03:16:00+09:00 + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/src/StellaOps.Feedser.Source.Jvn/Schemas/mod_sec_3.0.xsd b/src/StellaOps.Feedser.Source.Jvn/Schemas/mod_sec_3.0.xsd new file mode 100644 index 00000000..4ec18741 --- /dev/null +++ b/src/StellaOps.Feedser.Source.Jvn/Schemas/mod_sec_3.0.xsd @@ -0,0 +1,168 @@ + + + + + + + + + + + + + + mod_sec describes RSS Extension of security information + distribution, and definition of the tags for RSS 1.0, 2.0 and Atom. + mod_sec は、脆弱性対策情報などのセキュリティ情報を記述するための JVNRSS 拡張仕様で、RSS + 1.0、RSS 2.0、Atom での利用を想定した汎用的な仕様となっています。 + https://jvndb.jvn.jp/en/schema/mod_sec.html + https://jvndb.jvn.jp/schema/mod_sec.html + + Qualified Security Advisory Reference (mod_sec) + Masato Terada + 3.0 + 2017-07-20T03:16:00+09:00 + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + Specifies the relevant handling guidance for this STIX_Package. The + valid marking scope is the nearest STIXPackageType ancestor of this Handling element + and all its descendants. + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/src/StellaOps.Feedser.Source.Jvn/Schemas/status_3.3.xsd b/src/StellaOps.Feedser.Source.Jvn/Schemas/status_3.3.xsd new file mode 100644 index 00000000..85d86bea --- /dev/null +++ b/src/StellaOps.Feedser.Source.Jvn/Schemas/status_3.3.xsd @@ -0,0 +1,574 @@ + + + + + + + + + + + + + This is an XML Schema for the status information of MyJVN API. + MyJVN API のステータス情報を格納する XML スキーマ + + Status Information of MyJVN API + Masato Terada + 3.3 + 2017-07-20T03:16:00+09:00 + + + + + + + + + + + + + + + + + + Response Parameter; MyJVN API Schema Version - MyJVN API Ver 3.0 [common] + レスポンスパラメタ; MyJVN API スキーマバージョン - MyJNV API Ver 3.0 [共通] + + + + + Response Parameter; Return Code/Interger (0:success, 1:failure) [common] + レスポンスパラメタ; リターンコード/整数値 (0:成功, 1:エラー) [共通] + + + + + Request Parameter; Maximum number of Entry/Interger [common] + リクエストパラメタ; エントリ上限値/整数値 (APIごとに規定されている一度に取得できるエントリ件数の上限値, エラー時は空文字列) [共通] + + + + + Response Parameter; Error Code (Null:success) [common] + レスポンスパラメタ; エラーコード (空文字列:成功) [共通] + + + + + Response Parameter; Error Message (Null:success) [common] + レスポンスパラメタ; エラーメッセージ (空文字列:成功) [共通] + + + + + Response Parameter; Total number of Result entries [common] + レスポンスパラメタ; 応答エントリ総数: 整数値 (フィルタリング条件に当てはまるエントリの総件数) ；エラー時は空文字列 [共通] + + + + + Response Parameter; Number of Result entries [common] + レスポンスパラメタ; 応答エントリ数: 整数値 (フィルタリング条件に当てはまるエントリのうち、レスポンスに格納されている件数) ；エラー時は空文字列 [共通] + + + + + Response Parameter; Start entry number in Result entries [common] + レスポンスパラメタ; 応答エントリ開始位置: 整数値 (フィルタリング条件に当てはまるエントリのうち、何番目からのデータを取得したのかを示す値) ；エラー時は空文字列 [共通] + + + + + + + + + Request Parameter; Method [common] + リクエストパラメタ; メソッド名 [共通] + + + + + Request Parameter; Language (ja/en) [common] + リクエストパラメタ; 表示言語 (ja/en) [共通] + + + + + Request Parameter: Start entry number [common] + リクエストパラメタ: エントリ開始位置 [共通] + + + + + Request Parameter: Read entry number [common] + リクエストパラメタ: エントリ取得件数 [共通] + + + + + Request Parameter: XSL file enable/disable [common] + リクエストパラメタ: XSL ファイル 適用／未適用 [共通] + + + + + Request Parameter: feed name + リクエストパラメタ: フェードフォーマット(=APIバージョン)を示す名称 + + + + + + + + + Request Parameter: Vendor CPE Name/Product CPE Name + リクエストパラメタ: ベンダ CPE 名/製品 CPE 名 + + + + + Request Parameter: Vendor unique numbers + リクエストパラメタ: ベンダの識別番号一覧 + + + + + Request Parameter: Product unique numbers + リクエストパラメタ: 製品の識別番号一覧 + + + + + Request Parameter: Keyword + リクエストパラメタ: キーワード + + + + + Request Parameter: Type of OVAL + リクエストパラメタ: OVAL 種別 + method=getOvalList, getVulnOverviewStatistics + + + + + Request Parameter: Type of feed limit + リクエストパラメタ: フィード制限タイプ + method=getVendorList, getProductList,getVulnOverviewList, getVulnDetailInfo + + + + + + + + + Request Parameter: Product type (01/02/03) + リクエストパラメタ: 製品タイプ (01/02/03) + method=getProductList + + + + + Request Parameter: MyJVN API Version + リクエストパラメタ: MyJVN API Version + method=getProductList + + + + + Response Paramter; ReLatest date of product registration/update + レスポンスパラメタ: 製品登録/更新の最新日 + method=getProductList + + + + + + + + + Request Parameter: Severity + リクエストパラメタ: CVSS 深刻度 + + + + + Request Parameter: Vector of CVSS Base metric + リクエストパラメタ: CVSS 基本評価基準ベクタ + + + + + Request Parameter: Range of Date Public + リクエストパラメタ: 発見日の範囲指定 + + + + + Request Parameter: Range of Date Last Updated + リクエストパラメタ: 更新日の範囲指定 + + + + + Request Parameter: Range of Date First Published + リクエストパラメタ: 発行日の範囲指定 + + + + + Request Parameter: Start year of Date Public + リクエストパラメタ: 発見日開始年 + method=getVulnOverviewList + method=getStatistics + + + + + Request Parameter: Start month of Date Public + リクエストパラメタ: 発見日開始月 + method=getVulnOverviewList + method=getStatistics + + + + + Request Parameter: Start day of Date Public + リクエストパラメタ: 発見日開始日 + + + + + Request Parameter: End year of Date Public + リクエストパラメタ: 発見日終了年 + method=getVulnOverviewList + method=getStatistics + + + + + Request Parameter: End month of Date Public + リクエストパラメタ: 発見日終了月 + method=getVulnOverviewList + method=getStatistics + + + + + Request Parameter: End day of Date Public + リクエストパラメタ: 発見日終了日 + + + + + Request Parameter: Start year of Date Last Updated + リクエストパラメタ: 更新日開始年 + + + + + Request Parameter: Star month of Date Last Updated + リクエストパラメタ: 更新日開始月 + + + + + Request Parameter: Start day of Date Last Updated + リクエストパラメタ: 更新日開始日 + + + + + Request Parameter: End year of Date Last Updated + リクエストパラメタ: 更新日終了年 + + + + + Request Parameter: End month of Date Last Updated + リクエストパラメタ: 更新日終了月 + + + + + Request Parameter: End day of Date Last Updated + リクエストパラメタ: 更新日終了日 + + + + + Request Parameter: Start year of Date First Published + リクエストパラメタ: 発行日開始年 + + + + + Request Parameter: Start month of Date First Published + リクエストパラメタ: 発行日開始月 + + + + + Request Parameter: Start day of Date First Published + リクエストパラメタ: 発行日開始日 + + + + + Request Parameter: End year of Date First Published + リクエストパラメタ: 発行日終了年 + + + + + Request Parameter: End month of Date First Published + リクエストパラメタ: 発行日終了月 + + + + + Request Parameter: End day of Date First Published + リクエストパラメタ: 発行日終了日 + + + + + + + + + Request Parameter: Vulnerability ID + リクエストパラメタ: 脆弱性対策情報 ID + method=getVulnDetailInfo + + + + + + + + + Request Parameter: Vulnerability ID + リクエストパラメタ: 脆弱性対策情報 ID + method=getCvrfInfo + + + + + + + + + Request Parameter: Type of OS + リクエストパラメタ: OS 種別 + method=getOvalList + + + + + Request Parameter: Type of OVAL definition + リクエストパラメタ: OVAL定義のタイプ + method=getOvalList + + + + + Request Parameter: Type of Application condition + リクエストパラメタ: アプリケーションの動作モード + method=getOvalList + + + + + + + + + Request Parameter: OVAL ID + リクエストパラメタ: OVAL ID + method=getOvalData + + + + + + + + + Request Parameter: Benchmark ID + リクエストパラメタ: ベンチマーク ID + method=getXccdfCheckData + + + + + + + + + Request Parameter: Graph theme + リクエストパラメタ: グラフ テーマ + method=getStatistics + + + + + Response Parameter: Maxium number of cntAll + レスポンスパラメタ: cntAll の最大値 + method=getStatistics + + + + + Request Parameter: CWE ID + リクエストパラメタ: CWE 識別子 + method=getStatistics + + + + + Request Parameter: Product unique numbers + リクエストパラメタ: 製品の識別番号一覧 + + + + + + + + Request Parameter: reference + リクエストパラメタ: 参考情報 + method=getCPEdictionary + + + + + + + + Request Parameter: Date Last Updated (Year 4digits) + リクエストパラメタ: 更新日年 + method=getAlertList + + + + + + Request Parameter: Date First Published (Year 4digits) + リクエストパラメタ: 発行日年 + method=getAlertList + + + + + + Request Parameter: reference + リクエストパラメタ: 参考情報 + method=getAlertList + + + + + + + + + + + Define the version Number of Status XSD + Status XSD のバージョン番号 + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/src/StellaOps.Feedser.Source.Jvn/Schemas/tlp_marking.xsd b/src/StellaOps.Feedser.Source.Jvn/Schemas/tlp_marking.xsd new file mode 100644 index 00000000..97b9677c --- /dev/null +++ b/src/StellaOps.Feedser.Source.Jvn/Schemas/tlp_marking.xsd @@ -0,0 +1,40 @@ + + + + This schema was originally developed by The MITRE Corporation. The Data Marking Schema implementation is maintained by The MITRE Corporation and developed by the open STIX Community. For more information, including how to get involved in the effort and how to submit change requests, please visit the STIX website at http://stix.mitre.org. + + Data Marking Extension - TLP + 1.1.1 + 05/08/2014 9:00:00 AM + Data Marking Extension - TLP Marking Instance - Schematic implementation for attaching a Traffic Light Protocol (TLP)designation to an idendified XML structure. + Copyright (c) 2012-2014, The MITRE Corporation. All rights reserved. The contents of this file are subject to the terms of the STIX License located at http://stix.mitre.org/about/termsofuse.html. See the STIX License for the specific language governing permissions and limitations for use of this schema. When distributing copies of the STIX Schema, this license header must be included. + + + + + + The TLPMarkingStructureType is an implementation of the data marking schema that allows for a TLP Designation to be attached to an identified XML structure. Information about TLP is available here: http://www.us-cert.gov/tlp. + Nodes may be marked by multiple TLP Marking statements. When this occurs, the node should be considered marked at the most restrictive TLP Marking of all TLP Markings that were applied to it. For example, if a node is marked both GREEN and AMBER, the node should be considered AMBER. + + + + + + The TLP color designation of the marked structure. + + + + + + + + The TLP color designation of the marked structure. + + + + + + + + + diff --git a/src/StellaOps.Feedser.Source.Jvn/Schemas/vuldef_3.2.xsd b/src/StellaOps.Feedser.Source.Jvn/Schemas/vuldef_3.2.xsd new file mode 100644 index 00000000..4aeb27b4 --- /dev/null +++ b/src/StellaOps.Feedser.Source.Jvn/Schemas/vuldef_3.2.xsd @@ -0,0 +1,1566 @@ + + + + + + + + + + + + + This is an XML Schema for VULDEF - The Vulnerability Data + Publication and Exchange Format Data Model. + 脆弱性詳細情報の XML スキーマ VULDEF - The Vulnerability Data + Publication and Exchange Format Data Model + + VULDEF - The Vulnerability Data Publication and Exchange Format Data + Model + Masato Terada + 3.2 + 2017-07-20T03:16:00+09:00 + + + + + "VULDEF(The VULnerability Data publication and Exchange + Format data model)" is intended to be a format for the security information published by + the vendors or the Computer Security Incident Response Teams (CSIRTs). Assuming + widespread adoption of the VULDEF by the community, an organization can potentially + benefit from the increased automation in the processing of security advisory data, since + the commitment of vulnerability handling to parse free-form textual document will be + reduced. + "VULDEF(The VULnerability Data publication and Exchange + Format data model)" + の目的は、脆弱性情報ならびに脆弱性を除去するための脆弱性対策情報を提供し、流通させるために必要となるデータフォーマットを定義することにある。特に、脆弱性対策情報については、データフォーマットを定義することにより、情報自身の流通ならびに、関連対策情報同士の集約化を促すことができ、結果として対策促進を支援することができるであろう。 + + + The purpose of the "VULDEF(The VULnerability Data + publication and Exchange Format data model)" is to define data formats for information + related to security advisory typically published by the Vendors and Computer Security + Incident Response Teams (CSIRTs). An the Extensible Markup Language (XML) Document Type + Definition is developed, and examples are provided. + "VULDEF(The VULnerability Data publication and Exchange + Format data model)" では、脆弱性対策の情報提供(含む交換)において必要となる項目をデータモデルとして提示すると共に、XML + による表現形式を規定することにある。 + + + + + + + + + + + + VULDEF-Document class is the top level class in the + VULDEF data model and the DTD. All VULDEF documents are instances of the + VULDEF-Document class. The version of the VULDEF specification to which the VULDEF + document conforms. The value of this attribute MUST be 3.2. + VULDEFドキュメントクラスは、VULDEF データモデルと DTD のトップレベルのクラスである。全ての + VULDEF ドキュメントは、VULDEF ドキュメントクラスのインスタンスとなる。VULDEF のバージョン情報には "3.2" + を設定する。 + + + + + + + + + + + + + + + + + + + + In each publication of vulnerability related data is + represented by an instance of the Vulinfo class. This class provides a standardized + representation for commonly published vulnerability data and associates a unique + identifier. + Vulinfo + クラスは、脆弱性に関する情報(概要、想定される影響、対策など)を記載するクラスと、その脆弱性情報を一意に識別する識別子クラスから構成する。 + + + + + + + + + + + + + + + + + + VulinfoID class represents an vulnerability + information number that is unique in the context of the vendor or CSIRT and + identifies the activity characterized in an VULDEF-Document. A vulnerability number + assigned to this vulnerability information by the party that generated the document. + VulinfoID includes the organization prefix and unique number within the + organization. ex. {TA04-217A:US-CERT Alerts (CERT-TA)}{bid9835:Bugtraq + (BID)}{XF9324:ISS X-Force (XF)}{JVN54326:VN-JP (JVN)} + 脆弱性情報を一意に識別するための識別子であり、脆弱性情報を作成した組織が割り当てる。 + + + + + + + + + Group ID for vulnerability + information + 複数の脆弱性情報を取り扱う場合のグループ識別子を記載する。 + + + + + + + + + + + + + + + + + + + + + + The item(s) that constitute the vulnerability about + which the VULDEF-Document conveys information. The VulinfoData class summarizes the + details of the vulnerability information. + VulinfoData + クラスは、脆弱性情報として、脆弱性の概要、想定される影響、対策などの情報を記載する。 + + + + + + Title class describes the title of the + vulnerability information. + 脆弱性対策情報の題名を記載する。JVNRSS の item 要素の title + に対応する。 + + + + + + + + + + + + + + + + + + + + + + + + + + + VulinfoDescription class summarizes the detail of the + vulnerability information. + VulinfoDescription + クラスは、脆弱性に関する概要、技術的な解説、脆弱性のタイプの情報を記載する。 + + + + + + + + + + + + + + + + + + + + Overview is an abstract of the vulnerability that + provides a summary of the problem and its impact to the reader. + 脆弱性ならびにその対策に関する概要を記載する。JVNRSS1.0 の item 要素の description + に対応する。 + + + + + + + + + + + + + + + + + The vulnerability description contains one or more + paragraphs of text describing the vulnerability. + 脆弱性に関する詳細情報(技術的な解説)を記載する。 + + + + + + + + + + + + + + CWE + 脆弱性に関するタイプを記載する。 + + + + + + + + + + + + + + + + + Affected class includes vendors who may be affected + by the vulnerability. + Affected + クラスは、脆弱性により影響を受けるバージョン、システムに関する情報を提示するクラスである。 + + + + + + + + + + + + Entries in the Affected class. + 影響を受ける製品の項目 + + + + + + A vendor name of the affected + products. + 影響を受ける製品のベンダ名(提供者名)を記載する。 + + + + + + + + + A free-form textual description of the + affected products. + 影響を受ける製品に関する説明 + + + + + + + + + + + + A product name of the affected products. + 影響を受ける製品名を記載する。 + + + + + + + + + + + + + A version number of the affected products. + 影響を受ける製品のバージョンあるいはリビジョン番号を記載する。 + + + + + + + + + + + + + + A build number of the affected products. + 影響を受ける製品のビルド番号を記載する。 + + + + + + + + + + + + + + A version or build number of the affected products. + 影響を受ける製品のバージョン番号あるいはビルド番号の範囲を記載する。 + + + + + + + + + + + + + A version or revision number of the affected + products. + 影響を受ける製品のバージョン番号あるいはビルド番号の範囲を記載する。 + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + Impact class allows for classifying as well as + providing a description of the technical impact due to the + vulnerability. + Impact クラスは、脆弱性に伴い想定しうる影響を記載するクラスである。 + + + + + + + + + + + + + Cvss class is a information of the Common + Vulnerability Scoring System. + CVSS に関する情報を記載するクラスである。 + + + + + + + + + + + + + + + + + CVSS severity ranking. + CVSS 深刻度 + + + + + + + + + + + + + + CVSS Vector Strings. + CVSS 短縮表記 + + + + + + + + + + + + + CVSS Base Score. + CVSS 基本値 + + + + + + + + + + + + + CVSS Temporal Score. + CVSS 現状値 + + + + + + + + + + + + + CVSS Environmental Score. + CVSS 環境値 + + + + + + + + + + + + + Entries in the Impact class. + 想定される影響の項目 + + + + + + A free-form textual description of the + impact. + 想定される影響の項目に関する説明 + + + + + + + + + + + + + + + + + Solution class allows for classifying as well as + providing a description of the technical solution due to the + vulnerability. + Solution + クラスは、脆弱性の回避施策に関する情報を記載するクラスである。 + + + + + + + + + + + + Entries in the Solution class. + 脆弱性の回避施策の項目 + + + + + + A free-form textual description of the + solution. + 脆弱性の回避施策に関する説明 + + + + + + + + + + + + + + + + + Exploit class allows for classifying as well as + providing a description of the technical exploit due to the vulnerability. + Exploitクラスは、脆弱性の攻略に関する情報を記載するクラスである。 + + + + + + + + + + + + Entries in the Exploit class. + 脆弱性の攻略に関する項目 + + + + + + A free-form textual description of the + exploit. + 脆弱性の攻略に関する説明 + + + + + A URL to additional information about the + exploit. + 脆弱性の攻略に関する情報掲載 URL + + + + + + + + + + + + + + + + Related class is a collection of URLs at our web site + and others providing additional information about the vulnerability. + Relatedクラスは、参考情報など脆弱性に関連する情報を記載するクラスである。 + + + + + + + + + + + + Entries in the Related class. + 関連情報の項目を記載する。 + + + + + + A issuer of the + reference. + 脆弱性対策情報発行者の名称 + + + + + A ID of the reference. + 脆弱性対策情報を一意に識別するための識別子 + + + + + A title of the reference. + 脆弱性対策情報の題名 + + + + + A URL to related information about the + vulnerability. + 脆弱性対策情報の掲載 URL。JVNRSS の item 要素の dc:relation + に対応付ける。 + + + + + A free-form textual description of the + reference. + 関連情報の項目に関する説明 + + + + + + + + + + + + + + + + + Credit Class identifies who initially discovered the + vulnerability, anyone who was instrumental in the development of the document and + the contributors for anything. + + + + + + + + + + + + + Entries in the Credit class. + + + + + + + An author/contributor Name. + + + + + + A free-form textual description of the + credit. + + + + + + + + + + + + + + + + Contact class describes contact information of + VULDEF-Document issuer. + + + + + + + + + + + + + Entries in the Contact class. + + + + + + + + + + + + + + + + + + + + + + + + History class is a log or diary of the significant + events that occurred or actions performed by the issuers. + History クラスは、脆弱性情報の改訂履歴などを記載するクラスである。 + + + + + + + + + + + + HistoryItem class is a particular entry in the + History log that documents a particular significant action or + event. + 改訂履歴の項目 + + + + + + + + + + + + + + + + Number of the this entry in the history + log. + 改訂履歴の項目に付与する番号 + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + simpleType used when scoring on a scale of 0-10, + inclusive. + + + + + + + + + + + + + + + + + + + + + + + + + + + + + This attribute indicates the disclosure guidelines to + which the sender expects the recipient of the VULDEF-Document to adhere. This + attribute is defined as an enumerated value with a default value of + "private". + 送信側がVULDEF-Documentの受信側に期待する配布のガイドライン属性であり、以下の属性値(デフォルト値＝private)を選択する。 + + + + + There is no restriction level applied to the + information. + 情報配布に関する制約はない。 + + + + + The information may not be + shared. + 共有を期待する情報ではない。 + + + + + + + + The historyno attribute refers to HistoryNo class. + 改訂履歴の項目に付与する番号 + + + + + + + An estimate of the relative severity of the + vulnerability. The permitted values are shown below. There is no default value. + 脆弱性の相対的な深刻度の指標を、以下の属性値(デフォルト値＝なし)から選択する。 + + + + + + + + + + + Low severity. + 低 + + + + + Medium severity. + 中 + + + + + High severity. + 高 + + + + + + + + + + + + + + + This is the vulnerability information was known to + the public or not. + 脆弱性情報の公開状況を、以下の属性値(デフォルト値＝なし)から選択する。なお、配布のガイドライン属性restrictionとは、独立した属性である。 + + + + + Public information. + 公開済み + + + + + Not public information. + 未公開 + + + + + + + + + Each vulnerability in such a way that one can + understand the type of software problem that produced the + vulnerability. + 脆弱性のタイプを記載する。タイプとして、NIST NVD で使用している VulnerabilityType + を使用する。 + + + + + + + + + + + + + + + + + + + A vulnerability can enable either a "local" and/or + "remote" attack. + + + + + + The remote attack is possible. + + + + + + Need the account and logon operation. + + + + + + Both attacks are possible. + + + + + + + + + + This attribute indicates whether product is + vulnerable or not. There is no default value. + 影響を受ける製品毎の項目フィールドであり、下記に示す脆弱性の影響有無を記述するaffectedstatus + 属性を持っている。 + + + + + Vulnerable to the issue. + 影響あり + + + + + Not Vulnerable to the + issue. + 影響なし + + + + + Under investigation or a status can't be + fixed. + 不明 + + + + + Vulnerable to the issue and continue to + investigate. + 影響あり調査中 + + + + + Not Vulnerable to the issue and continue to + investigate. + 影響なし調査中 + + + + + + + + + This attribute is Comparison operators for a version + or build number. + + + + + + + + + + + + + + + + The type of impact in relatively broad categories. + The permitted values are shown below. + 想定される影響のタイプを記載する。タイプとして、IODEF で使用している Impacttype + 属性を使用する。 + + + + + Administrative privileges were attempted or + obtained. + + + + + + A denial of service was attempted or + completed. + + + + + + An action on a file was attempted or + completed. + + + + + + A reconnaissance probe was attempted or + completed. + + + + + + User privileges were attempted or + obtained. + + + + + + The activity did not have any (technical) + impact. + + + + + + The impact of the activity is unknown. + + + + + + Anything not in one of the above + categories. + + + + + + + + + + The type of solution in relatively broad categories. + There is no default value. + 回避施策のタイプを、以下の属性値(デフォルト値＝なし)から選択する。 + + + + + This solution eliminates the vulnerability. + 脆弱性そのものを除去する施策である。 + + + + + workaround solution (which has a direct + effect to resolve the issue). + 暫定施策(直接的な効果)である。 + + + + + migration solution (which has a indirect + effect to resolve the issue). + 緩和施策(間接的な効果)である。 + + + + + There is no solution. + 回避施策はない。 + + + + + Under investigation or a status can't be + fixed. + 不明(調査中など) + + + + + + + + + The type of exploit in relatively broad categories. + There is no default value. + + + + + + An exploit code exists. + すぐに悪用できるコードが存在する。 + + + + + POC exists. + 動作確認に利用できるコードが存在する。 + + + + + Worm, Virus or Trojan Hose + exists. + ワーム、ウイルス、トロイの木馬などのコードが存在する。 + + + + + Information for the exploit + exists. + 手順紹介レベルの情報が存在する。 + + + + + There are no exploits for this + issue. + 上記のいずれも存在しない。 + + + + + Currently we are not aware of any exploits + for this issue. + 不明 + + + + + + + + + The name of the database to which the reference is + being made. The permitted values are shown below. There is no default value. + 参照する情報源を以下の属性値(デフォルト値＝なし)から選択する。 + + + + + Bugtraq. (=Security + Focus.) + Bugtraq (=Security Focus) + + + + + Common Vulnerabilities and Exposures + (CVE). + Common Vulnerabilities and Exposures + (CVE) + + + + + CERT/CC Vulnerability Catalog. (=CERT + Advisory) + CERT/CC Vulnerability Catalog (=CERT + Advisory) + + + + + A product vendor. + 製品開発ベンダ + + + + + A local database. + + + + + + Comments by person. + + + + + + Except for the above. + 上記以外 + + + + + JVN. + JVN + + + + + JVN Status Tracking Notes. + JVN Status Tracking Notes + + + + + IPA Security Center + IPA セキュリティセンター 緊急対策情報 + + + + + + IPA セキュリティセンター + + + + + + JPCERT 緊急報告 + + + + + JPCERT Report. + JPCERT Report + + + + + @police topics + @police topics + + + + + CERT Advisory. + CERT Advisory + + + + + US-CERT Cyber Security + Alerts. + US-CERT Cyber Security Alerts + + + + + US-CERT Vulnerability + Note. + US-CERT Vulnerability Note + + + + + US-CERT Technical Cyber Security + Alert. + US-CERT Technical Cyber Security + Alert + + + + + National Vulnerability Database + (NVD). + National Vulnerability Database + (NVD) + + + + + CIAC Bulletins. + CIAC Bulletins + + + + + AUSCERT. + AUSCERT + + + + + NISCC Vulnerability + Advisory. + NISCC Vulnerability Advisory + + + + + Common Vulnerabilities and Exposures + (CVE). + Common Vulnerabilities and Exposures + (CVE) + + + + + Open Vulnerability and Assessment Language + (OVAL). + Open Vulnerability and Assessment Language + (OVAL) + + + + + Secunia Advisory. + Secunia Advisory + + + + + Security Focus. + Security Focus + + + + + ISS X-Force Database. + ISS X-Force Database + + + + + OPEN SOURCE VULNERABILITY DATABASE + (OSVDB). + OPEN SOURCE VULNERABILITY DATABASE + (OSVDB) + + + + + ISS Security Alerts and + Advisories. + ISS Security Alerts and + Advisories + + + + + + X-Force セキュリティアラート＆アドバイザリ + + + + + SecurityTracker. + SecurityTracker + + + + + SecuriTeam. + SecuriTeam + + + + + FrSIRT Advisories. + FrSIRT Advisories + + + + + The SANS Institute Diary. + The SANS Institute Diary + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/src/StellaOps.Feedser.Source.Jvn/Schemas/xml.xsd b/src/StellaOps.Feedser.Source.Jvn/Schemas/xml.xsd new file mode 100644 index 00000000..85bea790 --- /dev/null +++ b/src/StellaOps.Feedser.Source.Jvn/Schemas/xml.xsd @@ -0,0 +1,287 @@ + + + + + + +
    +

    About the XML namespace

    + +
    +

    + This schema document describes the XML namespace, in a form + suitable for import by other schema documents. +

    +

    + See + http://www.w3.org/XML/1998/namespace.html and + + http://www.w3.org/TR/REC-xml for information + about this namespace. +

    +

    + Note that local names in this namespace are intended to be + defined only by the World Wide Web Consortium or its subgroups. + The names currently defined in this namespace are listed below. + They should not be used with conflicting semantics by any Working + Group, specification, or document instance. +

    +

    + See further below in this document for more information about how to refer to this schema document from your own + XSD schema documents and about the + namespace-versioning policy governing this schema document. +

    +
    +
    +     +     + + + + +
    + +

    lang (as an attribute name)

    +

    + denotes an attribute whose value + is a language code for the natural language of the content of + any element; its value is inherited. This name is reserved + by virtue of its definition in the XML specification.

    + +
    +
    +

    Notes

    +

    + Attempting to install the relevant ISO 2- and 3-letter + codes as the enumerated possible values is probably never + going to be a realistic possibility. +

    +

    + See BCP 47 at + http://www.rfc-editor.org/rfc/bcp/bcp47.txt + and the IANA language subtag registry at + + http://www.iana.org/assignments/language-subtag-registry + for further information. +

    +

    + The union allows for the 'un-declaration' of xml:lang with + the empty string. +

    +
    +     +     + + + + + + + + + +     + + + + +
    + +

    space (as an attribute name)

    +

    + denotes an attribute whose + value is a keyword indicating what whitespace processing + discipline is intended for the content of the element; its + value is inherited. This name is reserved by virtue of its + definition in the XML specification.

    + +
    +     +     + + + + + + +     + + + +
    + +

    base (as an attribute name)

    +

    + denotes an attribute whose value + provides a URI to be used as the base for interpreting any + relative URIs in the scope of the element on which it + appears; its value is inherited. This name is reserved + by virtue of its definition in the XML Base specification.

    + +

    + See http://www.w3.org/TR/xmlbase/ + for information about this attribute. +

    +
    +     +     +     + + + + +
    + +

    id (as an attribute name)

    +

    + denotes an attribute whose value + should be interpreted as if declared to be of type ID. + This name is reserved by virtue of its definition in the + xml:id specification.

    + +

    + See http://www.w3.org/TR/xml-id/ + for information about this attribute. +

    +
    +     +     +     + + + + + + + + + + +
    + +

    Father (in any context at all)

    + +
    +

    + denotes Jon Bosak, the chair of + the original XML Working Group. This name is reserved by + the following decision of the W3C XML Plenary and + XML Coordination groups: +

    +
    +

    + In appreciation for his vision, leadership and + dedication the W3C XML Plenary on this 10th day of + February, 2000, reserves for Jon Bosak in perpetuity + the XML name "xml:Father". +

    +
    +
    +
    +     +     + + + +
    +

    About this schema document

    + +
    +

    + This schema defines attributes and an attribute group suitable + for use by schemas wishing to allow xml:base, + xml:lang, xml:space or + xml:id attributes on elements they define. +

    +

    + To enable this, such a schema must import this schema for + the XML namespace, e.g. as follows: +

    + 
    +          <schema . . .>
+           . . .
+           <import namespace="http://www.w3.org/XML/1998/namespace"
+                      schemaLocation="http://www.w3.org/2001/xml.xsd"/>
+
    +

    + or +

    + 
    +           <import namespace="http://www.w3.org/XML/1998/namespace"
+                      schemaLocation="http://www.w3.org/2009/01/xml.xsd"/>
+
    +

    + Subsequently, qualified reference to any of the attributes or the + group defined below will have the desired effect, e.g. +

    + 
    +          <type . . .>
+           . . .
+           <attributeGroup ref="xml:specialAttrs"/>
+
    +

    + will define a type which will schema-validate an instance element + with any of those attributes. +

    +
    +
    +     +     + + + +
    +

    Versioning policy for this schema document

    +
    +

    + In keeping with the XML Schema WG's standard versioning + policy, this schema document will persist at + + http://www.w3.org/2009/01/xml.xsd. +

    +

    + At the date of issue it can also be found at + + http://www.w3.org/2001/xml.xsd. +

    +

    + The schema document at that URI may however change in the future, + in order to remain compatible with the latest version of XML + Schema itself, or with the XML namespace itself. In other words, + if the XML Schema or XML namespaces change, the version of this + document at + http://www.w3.org/2001/xml.xsd + + will change accordingly; the version at + + http://www.w3.org/2009/01/xml.xsd + + will not change. +

    +

    + Previous dated (and unchanging) versions of this schema + document are at: +

    + +
    +
    +     +     + +     + diff --git a/src/StellaOps.Feedser.Source.Jvn/StellaOps.Feedser.Source.Jvn.csproj b/src/StellaOps.Feedser.Source.Jvn/StellaOps.Feedser.Source.Jvn.csproj new file mode 100644 index 00000000..c6e627a2 --- /dev/null +++ b/src/StellaOps.Feedser.Source.Jvn/StellaOps.Feedser.Source.Jvn.csproj @@ -0,0 +1,15 @@ + + + net10.0 + enable + enable + + + + + + + + + + diff --git a/src/StellaOps.Feedser.Source.Jvn/TASKS.md b/src/StellaOps.Feedser.Source.Jvn/TASKS.md new file mode 100644 index 00000000..31e3b3c8 --- /dev/null +++ b/src/StellaOps.Feedser.Source.Jvn/TASKS.md @@ -0,0 +1,13 @@ +# TASKS +| Task | Owner(s) | Depends on | Notes | +|---|---|---|---| +|MyJVN client (JVNRSS+VULDEF) with windowing|BE-Conn-JVN|Source.Common|**DONE** – windowed overview/detail fetch with rate limit handling implemented.| +|Schema/XSD validation and DTO sanitizer|BE-Conn-JVN, QA|Source.Common|**DONE** – parser validates XML against schema before DTO persistence.| +|Canonical mapping (aliases, jp_flags, refs)|BE-Conn-JVN|Models|**DONE** – mapper populates aliases, jp_flags, references while skipping non-actionable affected entries.| +|SourceState and idempotent dedupe|BE-Conn-JVN|Storage.Mongo|**DONE** – cursor tracks pending docs/mappings with resume support.| +|Golden fixtures and determinism tests|QA|Source.Jvn|**DONE** – deterministic snapshot test in `JvnConnectorTests` now passes with offline fixtures.| +|Async-safe overview query building|BE-Conn-JVN|Source.Common|DONE – `MyJvnClient` now builds query strings synchronously without blocking calls.| +|Reference dedupe + deterministic ordering|BE-Conn-JVN|Models|DONE – mapper merges by URL, retains richer metadata, sorts deterministically.| +|Console logging remediation|BE-Conn-JVN|Observability|**DONE** – connector now uses structured `ILogger` debug entries instead of console writes.| +|Offline fixtures for connector tests|QA|Source.Jvn|**DONE** – tests rely solely on canned HTTP responses and local fixtures.| +|Update VULDEF schema for vendor attribute|BE-Conn-JVN, QA|Source.Jvn|**DONE** – embedded XSD updated (vendor/product attrs, impact item), parser tightened, fixtures & snapshots refreshed.| diff --git a/src/StellaOps.Feedser.Source.Kev.Tests/Kev/Fixtures/kev-advisories.snapshot.json b/src/StellaOps.Feedser.Source.Kev.Tests/Kev/Fixtures/kev-advisories.snapshot.json new file mode 100644 index 00000000..d9802b3a --- /dev/null +++ b/src/StellaOps.Feedser.Source.Kev.Tests/Kev/Fixtures/kev-advisories.snapshot.json @@ -0,0 +1,271 @@ +[ + { + "advisoryKey": "kev/cve-2021-43798", + "affectedPackages": [ + { + "identifier": "Grafana Labs::Grafana", + "platform": null, + "provenance": [ + { + "fieldMask": [], + "kind": "mapping", + "recordedAt": "2025-10-10T00:01:00+00:00", + "source": "kev", + "value": "2025.10.09" + } + ], + "statuses": [], + "type": "vendor", + "versionRanges": [ + { + "fixedVersion": null, + "introducedVersion": null, + "lastAffectedVersion": null, + "primitives": { + "evr": null, + "hasVendorExtensions": true, + "nevra": null, + "semVer": null, + "vendorExtensions": { + "kev.vendorProject": "Grafana Labs", + "kev.product": "Grafana", + "kev.requiredAction": "Apply mitigations per vendor instructions, follow applicable BOD 22-01 guidance for cloud services, or discontinue use of the product if mitigations are unavailable.", + "kev.knownRansomwareCampaignUse": "Unknown", + "kev.notes": "https://grafana.com/security/advisory; https://nvd.nist.gov/vuln/detail/CVE-2021-43798", + "kev.catalogVersion": "2025.10.09", + "kev.catalogReleased": "2025-10-09T16:52:28.6547000+00:00", + "kev.dateAdded": "2025-10-09", + "kev.dueDate": "2025-10-30", + "kev.cwe": "CWE-22" + } + }, + "provenance": { + "fieldMask": [], + "kind": "kev-range", + "recordedAt": "2025-10-10T00:01:00+00:00", + "source": "kev", + "value": "Grafana Labs::Grafana" + }, + "rangeExpression": null, + "rangeKind": "vendor" + } + ] + } + ], + "aliases": [ + "CVE-2021-43798" + ], + "cvssMetrics": [], + "exploitKnown": true, + "language": "en", + "modified": "2025-10-09T16:52:28.6547+00:00", + "provenance": [ + { + "fieldMask": [], + "kind": "document", + "recordedAt": "2025-10-10T00:00:00+00:00", + "source": "kev", + "value": "https://www.cisa.gov/sites/default/files/feeds/known_exploited_vulnerabilities.json" + }, + { + "fieldMask": [], + "kind": "mapping", + "recordedAt": "2025-10-10T00:01:00+00:00", + "source": "kev", + "value": "2025.10.09" + } + ], + "published": "2025-10-09T00:00:00+00:00", + "references": [ + { + "kind": "reference", + "provenance": { + "fieldMask": [], + "kind": "reference", + "recordedAt": "2025-10-10T00:01:00+00:00", + "source": "kev", + "value": "CVE-2021-43798" + }, + "sourceTag": "kev.notes", + "summary": null, + "url": "https://grafana.com/security/advisory" + }, + { + "kind": "reference", + "provenance": { + "fieldMask": [], + "kind": "reference", + "recordedAt": "2025-10-10T00:01:00+00:00", + "source": "kev", + "value": "CVE-2021-43798" + }, + "sourceTag": "kev.notes", + "summary": null, + "url": "https://nvd.nist.gov/vuln/detail/CVE-2021-43798" + }, + { + "kind": "advisory", + "provenance": { + "fieldMask": [], + "kind": "reference", + "recordedAt": "2025-10-10T00:01:00+00:00", + "source": "kev", + "value": "CVE-2021-43798" + }, + "sourceTag": "cisa-kev", + "summary": null, + "url": "https://www.cisa.gov/known-exploited-vulnerabilities-catalog?search=CVE-2021-43798" + }, + { + "kind": "reference", + "provenance": { + "fieldMask": [], + "kind": "reference", + "recordedAt": "2025-10-10T00:01:00+00:00", + "source": "kev", + "value": "CVE-2021-43798" + }, + "sourceTag": "cisa-kev-feed", + "summary": null, + "url": "https://www.cisa.gov/sites/default/files/feeds/known_exploited_vulnerabilities.json" + } + ], + "severity": null, + "summary": "Grafana contains a path traversal vulnerability that could allow access to local files.", + "title": "Grafana Path Traversal Vulnerability" + }, + { + "advisoryKey": "kev/cve-2024-12345", + "affectedPackages": [ + { + "identifier": "Acme Corp::Acme Widget", + "platform": null, + "provenance": [ + { + "fieldMask": [], + "kind": "mapping", + "recordedAt": "2025-10-10T00:01:00+00:00", + "source": "kev", + "value": "2025.10.09" + } + ], + "statuses": [], + "type": "vendor", + "versionRanges": [ + { + "fixedVersion": null, + "introducedVersion": null, + "lastAffectedVersion": null, + "primitives": { + "evr": null, + "hasVendorExtensions": true, + "nevra": null, + "semVer": null, + "vendorExtensions": { + "kev.vendorProject": "Acme Corp", + "kev.product": "Acme Widget", + "kev.requiredAction": "Apply vendor patch KB-1234.", + "kev.knownRansomwareCampaignUse": "Confirmed", + "kev.notes": "https://acme.example/advisories/KB-1234 https://nvd.nist.gov/vuln/detail/CVE-2024-12345 additional context ignored", + "kev.catalogVersion": "2025.10.09", + "kev.catalogReleased": "2025-10-09T16:52:28.6547000+00:00", + "kev.dateAdded": "2025-08-01", + "kev.cwe": "CWE-120,CWE-787" + } + }, + "provenance": { + "fieldMask": [], + "kind": "kev-range", + "recordedAt": "2025-10-10T00:01:00+00:00", + "source": "kev", + "value": "Acme Corp::Acme Widget" + }, + "rangeExpression": null, + "rangeKind": "vendor" + } + ] + } + ], + "aliases": [ + "CVE-2024-12345" + ], + "cvssMetrics": [], + "exploitKnown": true, + "language": "en", + "modified": "2025-10-09T16:52:28.6547+00:00", + "provenance": [ + { + "fieldMask": [], + "kind": "document", + "recordedAt": "2025-10-10T00:00:00+00:00", + "source": "kev", + "value": "https://www.cisa.gov/sites/default/files/feeds/known_exploited_vulnerabilities.json" + }, + { + "fieldMask": [], + "kind": "mapping", + "recordedAt": "2025-10-10T00:01:00+00:00", + "source": "kev", + "value": "2025.10.09" + } + ], + "published": "2025-08-01T00:00:00+00:00", + "references": [ + { + "kind": "reference", + "provenance": { + "fieldMask": [], + "kind": "reference", + "recordedAt": "2025-10-10T00:01:00+00:00", + "source": "kev", + "value": "CVE-2024-12345" + }, + "sourceTag": "kev.notes", + "summary": null, + "url": "https://acme.example/advisories/KB-1234" + }, + { + "kind": "reference", + "provenance": { + "fieldMask": [], + "kind": "reference", + "recordedAt": "2025-10-10T00:01:00+00:00", + "source": "kev", + "value": "CVE-2024-12345" + }, + "sourceTag": "kev.notes", + "summary": null, + "url": "https://nvd.nist.gov/vuln/detail/CVE-2024-12345" + }, + { + "kind": "advisory", + "provenance": { + "fieldMask": [], + "kind": "reference", + "recordedAt": "2025-10-10T00:01:00+00:00", + "source": "kev", + "value": "CVE-2024-12345" + }, + "sourceTag": "cisa-kev", + "summary": null, + "url": "https://www.cisa.gov/known-exploited-vulnerabilities-catalog?search=CVE-2024-12345" + }, + { + "kind": "reference", + "provenance": { + "fieldMask": [], + "kind": "reference", + "recordedAt": "2025-10-10T00:01:00+00:00", + "source": "kev", + "value": "CVE-2024-12345" + }, + "sourceTag": "cisa-kev-feed", + "summary": null, + "url": "https://www.cisa.gov/sites/default/files/feeds/known_exploited_vulnerabilities.json" + } + ], + "severity": null, + "summary": "Acme Widget contains a buffer overflow that may allow remote code execution.", + "title": "Acme Widget Buffer Overflow" + } +] \ No newline at end of file diff --git a/src/StellaOps.Feedser.Source.Kev.Tests/Kev/Fixtures/kev-catalog.json b/src/StellaOps.Feedser.Source.Kev.Tests/Kev/Fixtures/kev-catalog.json new file mode 100644 index 00000000..f9733e7d --- /dev/null +++ b/src/StellaOps.Feedser.Source.Kev.Tests/Kev/Fixtures/kev-catalog.json @@ -0,0 +1,38 @@ +{ + "title": "CISA Catalog of Known Exploited Vulnerabilities", + "catalogVersion": "2025.10.09", + "dateReleased": "2025-10-09T16:52:28.6547Z", + "count": 2, + "vulnerabilities": [ + { + "cveID": "CVE-2021-43798", + "vendorProject": "Grafana Labs", + "product": "Grafana", + "vulnerabilityName": "Grafana Path Traversal Vulnerability", + "dateAdded": "2025-10-09", + "shortDescription": "Grafana contains a path traversal vulnerability that could allow access to local files.", + "requiredAction": "Apply mitigations per vendor instructions, follow applicable BOD 22-01 guidance for cloud services, or discontinue use of the product if mitigations are unavailable.", + "dueDate": "2025-10-30", + "knownRansomwareCampaignUse": "Unknown", + "notes": "https://grafana.com/security/advisory; https://nvd.nist.gov/vuln/detail/CVE-2021-43798", + "cwes": [ + "CWE-22" + ] + }, + { + "cveID": "CVE-2024-12345", + "vendorProject": "Acme Corp", + "product": "Acme Widget", + "vulnerabilityName": "Acme Widget Buffer Overflow", + "dateAdded": "2025-08-01", + "shortDescription": "Acme Widget contains a buffer overflow that may allow remote code execution.", + "requiredAction": "Apply vendor patch KB-1234.", + "knownRansomwareCampaignUse": "Confirmed", + "notes": "https://acme.example/advisories/KB-1234 https://nvd.nist.gov/vuln/detail/CVE-2024-12345 additional context ignored", + "cwes": [ + "CWE-120", + "CWE-787" + ] + } + ] +} diff --git a/src/StellaOps.Feedser.Source.Kev.Tests/Kev/KevConnectorTests.cs b/src/StellaOps.Feedser.Source.Kev.Tests/Kev/KevConnectorTests.cs new file mode 100644 index 00000000..d681011c --- /dev/null +++ b/src/StellaOps.Feedser.Source.Kev.Tests/Kev/KevConnectorTests.cs @@ -0,0 +1,218 @@ +using System.Net; +using System.Net.Http; +using System.Net.Http.Headers; +using System.Text; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Http; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Logging.Abstractions; +using Microsoft.Extensions.Time.Testing; +using MongoDB.Bson; +using StellaOps.Feedser.Models; +using StellaOps.Feedser.Source.Common; +using StellaOps.Feedser.Source.Common.Http; +using StellaOps.Feedser.Source.Common.Testing; +using StellaOps.Feedser.Source.Kev; +using StellaOps.Feedser.Source.Kev.Configuration; +using StellaOps.Feedser.Storage.Mongo; +using StellaOps.Feedser.Storage.Mongo.Advisories; +using StellaOps.Feedser.Storage.Mongo.Documents; +using StellaOps.Feedser.Testing; +using Xunit; + +namespace StellaOps.Feedser.Source.Kev.Tests; + +[Collection("mongo-fixture")] +public sealed class KevConnectorTests : IAsyncLifetime +{ + private static readonly Uri FeedUri = new("https://www.cisa.gov/sites/default/files/feeds/known_exploited_vulnerabilities.json"); + private const string CatalogEtag = "\"kev-2025-10-09\""; + + private readonly MongoIntegrationFixture _fixture; + private readonly FakeTimeProvider _timeProvider; + private readonly CannedHttpMessageHandler _handler; + + public KevConnectorTests(MongoIntegrationFixture fixture) + { + _fixture = fixture; + _timeProvider = new FakeTimeProvider(new DateTimeOffset(2025, 10, 10, 0, 0, 0, TimeSpan.Zero)); + _handler = new CannedHttpMessageHandler(); + } + + [Fact] + public async Task FetchParseMap_ProducesDeterministicSnapshot() + { + await using var provider = await BuildServiceProviderAsync(); + SeedCatalogResponse(); + + var connector = provider.GetRequiredService(); + await connector.FetchAsync(provider, CancellationToken.None); + _timeProvider.Advance(TimeSpan.FromMinutes(1)); + await connector.ParseAsync(provider, CancellationToken.None); + await connector.MapAsync(provider, CancellationToken.None); + + var advisoryStore = provider.GetRequiredService(); + var advisories = await advisoryStore.GetRecentAsync(10, CancellationToken.None); + Assert.NotEmpty(advisories); + + var ordered = advisories.OrderBy(static a => a.AdvisoryKey, StringComparer.Ordinal).ToArray(); + var snapshot = SnapshotSerializer.ToSnapshot(ordered); + WriteOrAssertSnapshot(snapshot, "kev-advisories.snapshot.json"); + + var documentStore = provider.GetRequiredService(); + var document = await documentStore.FindBySourceAndUriAsync(KevConnectorPlugin.SourceName, FeedUri.ToString(), CancellationToken.None); + Assert.NotNull(document); + Assert.Equal(DocumentStatuses.Mapped, document!.Status); + + SeedNotModifiedResponse(); + await connector.FetchAsync(provider, CancellationToken.None); + _handler.AssertNoPendingResponses(); + + var stateRepository = provider.GetRequiredService(); + var state = await stateRepository.TryGetAsync(KevConnectorPlugin.SourceName, CancellationToken.None); + Assert.NotNull(state); + Assert.Equal("2025.10.09", state!.Cursor.TryGetValue("catalogVersion", out var versionValue) ? versionValue.AsString : null); + Assert.True(state.Cursor.TryGetValue("catalogReleased", out var releasedValue) && releasedValue.BsonType is BsonType.DateTime); + Assert.True(IsEmptyArray(state.Cursor, "pendingDocuments")); + Assert.True(IsEmptyArray(state.Cursor, "pendingMappings")); + } + + private async Task BuildServiceProviderAsync() + { + await _fixture.Client.DropDatabaseAsync(_fixture.Database.DatabaseNamespace.DatabaseName); + _handler.Clear(); + + var services = new ServiceCollection(); + services.AddLogging(builder => builder.AddProvider(NullLoggerProvider.Instance)); + services.AddSingleton(_timeProvider); + + services.AddMongoStorage(options => + { + options.ConnectionString = _fixture.Runner.ConnectionString; + options.DatabaseName = _fixture.Database.DatabaseNamespace.DatabaseName; + options.CommandTimeout = TimeSpan.FromSeconds(5); + }); + + services.AddSourceCommon(); + services.AddKevConnector(options => + { + options.FeedUri = FeedUri; + options.RequestTimeout = TimeSpan.FromSeconds(10); + }); + + services.Configure(KevOptions.HttpClientName, builderOptions => + { + builderOptions.HttpMessageHandlerBuilderActions.Add(builder => builder.PrimaryHandler = _handler); + }); + + var provider = services.BuildServiceProvider(); + var bootstrapper = provider.GetRequiredService(); + await bootstrapper.InitializeAsync(CancellationToken.None); + return provider; + } + + private void SeedCatalogResponse() + { + var payload = ReadFixture("kev-catalog.json"); + _handler.AddResponse(FeedUri, () => + { + var response = new HttpResponseMessage(HttpStatusCode.OK) + { + Content = new StringContent(payload, Encoding.UTF8, "application/json"), + }; + response.Headers.ETag = new EntityTagHeaderValue(CatalogEtag); + response.Content.Headers.LastModified = new DateTimeOffset(2025, 10, 9, 16, 52, 28, TimeSpan.Zero); + return response; + }); + } + + private void SeedNotModifiedResponse() + { + _handler.AddResponse(FeedUri, () => + { + var response = new HttpResponseMessage(HttpStatusCode.NotModified); + response.Headers.ETag = new EntityTagHeaderValue(CatalogEtag); + return response; + }); + } + + private static bool IsEmptyArray(BsonDocument document, string field) + { + if (!document.TryGetValue(field, out var value) || value is not BsonArray array) + { + return false; + } + + return array.Count == 0; + } + + private static string ReadFixture(string filename) + { + var path = GetExistingFixturePath(filename); + return File.ReadAllText(path); + } + + private static void WriteOrAssertSnapshot(string snapshot, string filename) + { + if (ShouldUpdateFixtures()) + { + var target = GetWritableFixturePath(filename); + File.WriteAllText(target, snapshot); + return; + } + + var expected = ReadFixture(filename); + var normalizedExpected = Normalize(expected); + var normalizedSnapshot = Normalize(snapshot); + + if (!string.Equals(normalizedExpected, normalizedSnapshot, StringComparison.Ordinal)) + { + var actualPath = Path.Combine(Path.GetDirectoryName(GetWritableFixturePath(filename))!, Path.GetFileNameWithoutExtension(filename) + ".actual.json"); + File.WriteAllText(actualPath, snapshot); + } + + Assert.Equal(normalizedExpected, normalizedSnapshot); + } + + private static bool ShouldUpdateFixtures() + { + var value = Environment.GetEnvironmentVariable("UPDATE_KEV_FIXTURES"); + return string.Equals(value, "1", StringComparison.Ordinal) || string.Equals(value, "true", StringComparison.OrdinalIgnoreCase); + } + + private static string Normalize(string value) + => value.Replace("\r\n", "\n", StringComparison.Ordinal); + + private static string GetExistingFixturePath(string filename) + { + var baseDir = AppContext.BaseDirectory; + var primary = Path.Combine(baseDir, "Source", "Kev", "Fixtures", filename); + if (File.Exists(primary)) + { + return primary; + } + + var fallback = Path.Combine(baseDir, "Kev", "Fixtures", filename); + if (File.Exists(fallback)) + { + return fallback; + } + + throw new FileNotFoundException($"Unable to locate KEV fixture '{filename}'."); + } + + private static string GetWritableFixturePath(string filename) + { + var baseDir = AppContext.BaseDirectory; + var primaryDir = Path.Combine(baseDir, "Source", "Kev", "Fixtures"); + Directory.CreateDirectory(primaryDir); + return Path.Combine(primaryDir, filename); + } + + public Task InitializeAsync() => Task.CompletedTask; + + public async Task DisposeAsync() + { + await _fixture.Client.DropDatabaseAsync(_fixture.Database.DatabaseNamespace.DatabaseName); + } +} diff --git a/src/StellaOps.Feedser.Source.Kev.Tests/Kev/KevMapperTests.cs b/src/StellaOps.Feedser.Source.Kev.Tests/Kev/KevMapperTests.cs new file mode 100644 index 00000000..9737cf91 --- /dev/null +++ b/src/StellaOps.Feedser.Source.Kev.Tests/Kev/KevMapperTests.cs @@ -0,0 +1,70 @@ +using System; +using System.Linq; +using StellaOps.Feedser.Models; +using StellaOps.Feedser.Source.Kev; +using StellaOps.Feedser.Source.Kev.Internal; +using Xunit; + +namespace StellaOps.Feedser.Source.Kev.Tests; + +public sealed class KevMapperTests +{ + [Fact] + public void Map_BuildsVendorRangePrimitivesWithDueDate() + { + var catalog = new KevCatalogDto + { + CatalogVersion = "2025.10.09", + DateReleased = new DateTimeOffset(2025, 10, 9, 16, 52, 28, TimeSpan.Zero), + Vulnerabilities = new[] + { + new KevVulnerabilityDto + { + CveId = "CVE-2021-43798", + VendorProject = "Grafana Labs", + Product = "Grafana", + VulnerabilityName = "Grafana Path Traversal Vulnerability", + DateAdded = "2025-10-09", + ShortDescription = "Grafana contains a path traversal vulnerability that could allow access to local files.", + RequiredAction = "Apply mitigations per vendor instructions or discontinue use.", + DueDate = "2025-10-30", + KnownRansomwareCampaignUse = "Unknown", + Notes = "https://grafana.com/security/advisory; https://nvd.nist.gov/vuln/detail/CVE-2021-43798", + Cwes = new[] { "CWE-22" } + } + } + }; + + var feedUri = new Uri("https://www.cisa.gov/sites/default/files/feeds/known_exploited_vulnerabilities.json"); + var fetchedAt = new DateTimeOffset(2025, 10, 9, 17, 0, 0, TimeSpan.Zero); + var validatedAt = fetchedAt.AddMinutes(1); + + var advisories = KevMapper.Map(catalog, KevConnectorPlugin.SourceName, feedUri, fetchedAt, validatedAt); + + var advisory = Assert.Single(advisories); + Assert.True(advisory.ExploitKnown); + Assert.Contains("cve-2021-43798", advisory.Aliases, StringComparer.OrdinalIgnoreCase); + + var affected = Assert.Single(advisory.AffectedPackages); + Assert.Equal(AffectedPackageTypes.Vendor, affected.Type); + Assert.Equal("Grafana Labs::Grafana", affected.Identifier); + + var range = Assert.Single(affected.VersionRanges); + Assert.Equal(AffectedPackageTypes.Vendor, range.RangeKind); + var primitives = range.Primitives; + Assert.NotNull(primitives); + + Assert.True(primitives!.HasVendorExtensions); + var extensions = primitives!.VendorExtensions!; + Assert.Equal("Grafana Labs", extensions["kev.vendorProject"]); + Assert.Equal("Grafana", extensions["kev.product"]); + Assert.Equal("2025-10-30", extensions["kev.dueDate"]); + Assert.Equal("Unknown", extensions["kev.knownRansomwareCampaignUse"]); + Assert.Equal("CWE-22", extensions["kev.cwe"]); + + var references = advisory.References.Select(reference => reference.Url).ToArray(); + Assert.Contains("https://grafana.com/security/advisory", references); + Assert.Contains("https://nvd.nist.gov/vuln/detail/CVE-2021-43798", references); + Assert.Contains("https://www.cisa.gov/known-exploited-vulnerabilities-catalog?search=CVE-2021-43798", references); + } +} diff --git a/src/StellaOps.Feedser.Source.Kev.Tests/StellaOps.Feedser.Source.Kev.Tests.csproj b/src/StellaOps.Feedser.Source.Kev.Tests/StellaOps.Feedser.Source.Kev.Tests.csproj new file mode 100644 index 00000000..57e5921a --- /dev/null +++ b/src/StellaOps.Feedser.Source.Kev.Tests/StellaOps.Feedser.Source.Kev.Tests.csproj @@ -0,0 +1,19 @@ + + + net10.0 + enable + enable + + + + + + + + + + + + + + diff --git a/src/StellaOps.Feedser.Source.Kev/AGENTS.md b/src/StellaOps.Feedser.Source.Kev/AGENTS.md new file mode 100644 index 00000000..9665f59e --- /dev/null +++ b/src/StellaOps.Feedser.Source.Kev/AGENTS.md @@ -0,0 +1,44 @@ +# AGENTS +## Role +Implement the CISA Known Exploited Vulnerabilities (KEV) catalogue connector to ingest KEV entries for enrichment and policy checks. + +## Scope +- Integrate with the official KEV JSON feed; understand schema, update cadence, and pagination (if any). +- Implement fetch job with incremental updates, checksum validation, and cursor persistence. +- Parse KEV entries (CVE ID, vendor/product, required actions, due dates). +- Map entries into canonical `Advisory` (or augmentation) records with aliases, references, affected packages, and range primitives capturing enforcement metadata. +- Deliver deterministic fixtures and regression tests. + +## Participants +- `Source.Common` (HTTP client, fetch service, DTO storage). +- `Storage.Mongo` (raw/document/DTO/advisory stores, source state). +- `Feedser.Models` (advisory + range primitive types). +- `Feedser.Testing` (integration fixtures & snapshots). + +## Interfaces & Contracts +- Job kinds: `kev:fetch`, `kev:parse`, `kev:map`. +- Persist upstream `catalogLastUpdated` / ETag to detect changes. +- Alias list must include CVE ID; references should point to CISA KEV listing and vendor advisories. + +## In/Out of scope +In scope: +- KEV feed ingestion and canonical mapping. +- Range primitives capturing remediation due dates or vendor requirements. + +Out of scope: +- Compliance policy enforcement (handled elsewhere). + +## Observability & Security Expectations +- Log fetch timestamps, updated entry counts, and mapping stats. +- Handle data anomalies and record failures with backoff. +- Validate JSON payloads before persistence. +- Structured informational logs should surface the catalog version, release timestamp, and advisory counts for each successful parse/map cycle. + +## Operational Notes +- HTTP allowlist is limited to `www.cisa.gov`; operators should mirror / proxy that hostname for air-gapped deployments. +- CISA publishes KEV updates daily (catalogVersion follows `yyyy.MM.dd`). Expect releases near 16:30–17:00 UTC and retain overlap when scheduling fetches. + +## Tests +- Add `StellaOps.Feedser.Source.Kev.Tests` covering fetch/parse/map with KEV JSON fixtures. +- Snapshot canonical output; allow fixture regeneration via env flag. +- Ensure deterministic ordering/time normalisation. diff --git a/src/StellaOps.Feedser.Source.Kev/Configuration/KevOptions.cs b/src/StellaOps.Feedser.Source.Kev/Configuration/KevOptions.cs new file mode 100644 index 00000000..c8de4041 --- /dev/null +++ b/src/StellaOps.Feedser.Source.Kev/Configuration/KevOptions.cs @@ -0,0 +1,33 @@ +using System; +using System.Diagnostics.CodeAnalysis; + +namespace StellaOps.Feedser.Source.Kev.Configuration; + +public sealed class KevOptions +{ + public static string HttpClientName => "source.kev"; + + /// + /// Official CISA Known Exploited Vulnerabilities JSON feed. + /// + public Uri FeedUri { get; set; } = new("https://www.cisa.gov/sites/default/files/feeds/known_exploited_vulnerabilities.json", UriKind.Absolute); + + /// + /// Timeout applied to KEV feed requests. + /// + public TimeSpan RequestTimeout { get; set; } = TimeSpan.FromSeconds(30); + + [MemberNotNull(nameof(FeedUri))] + public void Validate() + { + if (FeedUri is null || !FeedUri.IsAbsoluteUri) + { + throw new InvalidOperationException("FeedUri must be an absolute URI."); + } + + if (RequestTimeout <= TimeSpan.Zero) + { + throw new InvalidOperationException("RequestTimeout must be greater than zero."); + } + } +} diff --git a/src/StellaOps.Feedser.Source.Kev/Internal/KevCatalogDto.cs b/src/StellaOps.Feedser.Source.Kev/Internal/KevCatalogDto.cs new file mode 100644 index 00000000..f786cc82 --- /dev/null +++ b/src/StellaOps.Feedser.Source.Kev/Internal/KevCatalogDto.cs @@ -0,0 +1,59 @@ +using System; +using System.Collections.Generic; +using System.Text.Json.Serialization; + +namespace StellaOps.Feedser.Source.Kev.Internal; + +internal sealed record KevCatalogDto +{ + [JsonPropertyName("title")] + public string? Title { get; init; } + + [JsonPropertyName("catalogVersion")] + public string? CatalogVersion { get; init; } + + [JsonPropertyName("dateReleased")] + public DateTimeOffset? DateReleased { get; init; } + + [JsonPropertyName("count")] + public int Count { get; init; } + + [JsonPropertyName("vulnerabilities")] + public IReadOnlyList Vulnerabilities { get; init; } = Array.Empty(); +} + +internal sealed record KevVulnerabilityDto +{ + [JsonPropertyName("cveID")] + public string? CveId { get; init; } + + [JsonPropertyName("vendorProject")] + public string? VendorProject { get; init; } + + [JsonPropertyName("product")] + public string? Product { get; init; } + + [JsonPropertyName("vulnerabilityName")] + public string? VulnerabilityName { get; init; } + + [JsonPropertyName("dateAdded")] + public string? DateAdded { get; init; } + + [JsonPropertyName("shortDescription")] + public string? ShortDescription { get; init; } + + [JsonPropertyName("requiredAction")] + public string? RequiredAction { get; init; } + + [JsonPropertyName("dueDate")] + public string? DueDate { get; init; } + + [JsonPropertyName("knownRansomwareCampaignUse")] + public string? KnownRansomwareCampaignUse { get; init; } + + [JsonPropertyName("notes")] + public string? Notes { get; init; } + + [JsonPropertyName("cwes")] + public IReadOnlyList Cwes { get; init; } = Array.Empty(); +} diff --git a/src/StellaOps.Feedser.Source.Kev/Internal/KevCursor.cs b/src/StellaOps.Feedser.Source.Kev/Internal/KevCursor.cs new file mode 100644 index 00000000..7a1028dd --- /dev/null +++ b/src/StellaOps.Feedser.Source.Kev/Internal/KevCursor.cs @@ -0,0 +1,103 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using MongoDB.Bson; + +namespace StellaOps.Feedser.Source.Kev.Internal; + +internal sealed record KevCursor( + string? CatalogVersion, + DateTimeOffset? CatalogReleased, + IReadOnlyCollection PendingDocuments, + IReadOnlyCollection PendingMappings) +{ + public static KevCursor Empty { get; } = new(null, null, Array.Empty(), Array.Empty()); + + public BsonDocument ToBsonDocument() + { + var document = new BsonDocument + { + ["pendingDocuments"] = new BsonArray(PendingDocuments.Select(static id => id.ToString())), + ["pendingMappings"] = new BsonArray(PendingMappings.Select(static id => id.ToString())), + }; + + if (!string.IsNullOrEmpty(CatalogVersion)) + { + document["catalogVersion"] = CatalogVersion; + } + + if (CatalogReleased.HasValue) + { + document["catalogReleased"] = CatalogReleased.Value.UtcDateTime; + } + + return document; + } + + public static KevCursor FromBson(BsonDocument? document) + { + if (document is null || document.ElementCount == 0) + { + return Empty; + } + + var version = document.TryGetValue("catalogVersion", out var versionValue) + ? versionValue.AsString + : null; + + var released = document.TryGetValue("catalogReleased", out var releasedValue) + ? ParseDate(releasedValue) + : null; + + return new KevCursor( + version, + released, + ReadGuidArray(document, "pendingDocuments"), + ReadGuidArray(document, "pendingMappings")); + } + + public KevCursor WithCatalogMetadata(string? version, DateTimeOffset? released) + => this with + { + CatalogVersion = string.IsNullOrWhiteSpace(version) ? null : version.Trim(), + CatalogReleased = released?.ToUniversalTime(), + }; + + public KevCursor WithPendingDocuments(IEnumerable ids) + => this with { PendingDocuments = ids?.Distinct().ToArray() ?? Array.Empty() }; + + public KevCursor WithPendingMappings(IEnumerable ids) + => this with { PendingMappings = ids?.Distinct().ToArray() ?? Array.Empty() }; + + private static DateTimeOffset? ParseDate(BsonValue value) + => value.BsonType switch + { + BsonType.DateTime => DateTime.SpecifyKind(value.ToUniversalTime(), DateTimeKind.Utc), + BsonType.String when DateTimeOffset.TryParse(value.AsString, out var parsed) => parsed.ToUniversalTime(), + _ => null, + }; + + private static IReadOnlyCollection ReadGuidArray(BsonDocument document, string field) + { + if (!document.TryGetValue(field, out var value) || value is not BsonArray array) + { + return Array.Empty(); + } + + var results = new List(array.Count); + foreach (var element in array) + { + if (element is null) + { + continue; + } + + if (Guid.TryParse(element.ToString(), out var guid)) + { + results.Add(guid); + } + } + + return results; + } +} diff --git a/src/StellaOps.Feedser.Source.Kev/Internal/KevDiagnostics.cs b/src/StellaOps.Feedser.Source.Kev/Internal/KevDiagnostics.cs new file mode 100644 index 00000000..c15b44a5 --- /dev/null +++ b/src/StellaOps.Feedser.Source.Kev/Internal/KevDiagnostics.cs @@ -0,0 +1,48 @@ +using System.Diagnostics.Metrics; + +namespace StellaOps.Feedser.Source.Kev.Internal; + +internal sealed class KevDiagnostics : IDisposable +{ + public const string MeterName = "StellaOps.Feedser.Source.Kev"; + private static readonly string MeterVersion = "1.0.0"; + + private readonly Meter _meter; + private readonly Counter _parsedEntries; + private readonly Counter _mappedAdvisories; + + public KevDiagnostics() + { + _meter = new Meter(MeterName, MeterVersion); + _parsedEntries = _meter.CreateCounter( + name: "kev.parse.entries", + unit: "entries", + description: "Number of KEV vulnerabilities parsed from the catalog."); + _mappedAdvisories = _meter.CreateCounter( + name: "kev.map.advisories", + unit: "advisories", + description: "Number of KEV advisories emitted during mapping."); + } + + public void CatalogParsed(string? catalogVersion, int entryCount) + { + if (entryCount <= 0) + { + return; + } + + _parsedEntries.Add(entryCount, new KeyValuePair("catalogVersion", catalogVersion ?? string.Empty)); + } + + public void AdvisoriesMapped(string? catalogVersion, int advisoryCount) + { + if (advisoryCount <= 0) + { + return; + } + + _mappedAdvisories.Add(advisoryCount, new KeyValuePair("catalogVersion", catalogVersion ?? string.Empty)); + } + + public void Dispose() => _meter.Dispose(); +} diff --git a/src/StellaOps.Feedser.Source.Kev/Internal/KevMapper.cs b/src/StellaOps.Feedser.Source.Kev/Internal/KevMapper.cs new file mode 100644 index 00000000..9088774a --- /dev/null +++ b/src/StellaOps.Feedser.Source.Kev/Internal/KevMapper.cs @@ -0,0 +1,324 @@ +using System; +using System.Collections.Generic; +using System.Globalization; +using System.Linq; +using System.Text; +using StellaOps.Feedser.Models; + +namespace StellaOps.Feedser.Source.Kev.Internal; + +internal static class KevMapper +{ + public static IReadOnlyList Map( + KevCatalogDto catalog, + string sourceName, + Uri feedUri, + DateTimeOffset fetchedAt, + DateTimeOffset validatedAt) + { + ArgumentNullException.ThrowIfNull(catalog); + ArgumentNullException.ThrowIfNull(sourceName); + ArgumentNullException.ThrowIfNull(feedUri); + + var advisories = new List(); + var fetchProvenance = new AdvisoryProvenance(sourceName, "document", feedUri.ToString(), fetchedAt); + var mappingProvenance = new AdvisoryProvenance( + sourceName, + "mapping", + catalog.CatalogVersion ?? feedUri.ToString(), + validatedAt); + + if (catalog.Vulnerabilities is null || catalog.Vulnerabilities.Count == 0) + { + return advisories; + } + + foreach (var entry in catalog.Vulnerabilities) + { + if (entry is null) + { + continue; + } + + var cveId = Normalize(entry.CveId); + if (string.IsNullOrEmpty(cveId)) + { + continue; + } + + var advisoryKey = $"kev/{cveId.ToLowerInvariant()}"; + var title = Normalize(entry.VulnerabilityName) ?? cveId; + var summary = Normalize(entry.ShortDescription); + var published = ParseDate(entry.DateAdded); + var dueDate = ParseDate(entry.DueDate); + + var aliases = new HashSet(StringComparer.OrdinalIgnoreCase) { cveId }; + + var references = BuildReferences(entry, sourceName, mappingProvenance, feedUri, cveId).ToArray(); + + var affectedPackages = BuildAffectedPackages( + entry, + catalog, + sourceName, + mappingProvenance, + published, + dueDate).ToArray(); + + var provenance = new[] + { + fetchProvenance, + mappingProvenance + }; + + advisories.Add(new Advisory( + advisoryKey, + title, + summary, + language: "en", + published, + modified: catalog.DateReleased?.ToUniversalTime(), + severity: null, + exploitKnown: true, + aliases, + references, + affectedPackages, + cvssMetrics: Array.Empty(), + provenance)); + } + + return advisories + .OrderBy(static advisory => advisory.AdvisoryKey, StringComparer.Ordinal) + .ToArray(); + } + + private static IEnumerable BuildReferences( + KevVulnerabilityDto entry, + string sourceName, + AdvisoryProvenance mappingProvenance, + Uri feedUri, + string cveId) + { + var references = new List(); + var provenance = new AdvisoryProvenance(sourceName, "reference", cveId, mappingProvenance.RecordedAt); + + var catalogUrl = BuildCatalogSearchUrl(cveId); + if (catalogUrl is not null) + { + TryAddReference(references, catalogUrl, "advisory", "cisa-kev", provenance); + } + + TryAddReference(references, feedUri.ToString(), "reference", "cisa-kev-feed", provenance); + + foreach (var url in ExtractUrls(entry.Notes)) + { + TryAddReference(references, url, "reference", "kev.notes", provenance); + } + + return references + .GroupBy(static r => r.Url, StringComparer.OrdinalIgnoreCase) + .Select(static group => group + .OrderBy(static r => r.Kind, StringComparer.Ordinal) + .ThenBy(static r => r.SourceTag, StringComparer.Ordinal) + .First()) + .OrderBy(static r => r.Kind, StringComparer.Ordinal) + .ThenBy(static r => r.Url, StringComparer.Ordinal) + .ToArray(); + } + + private static void TryAddReference( + ICollection references, + string? url, + string kind, + string? sourceTag, + AdvisoryProvenance provenance) + { + if (string.IsNullOrWhiteSpace(url)) + { + return; + } + + if (!Uri.TryCreate(url, UriKind.Absolute, out var parsed) + || (parsed.Scheme != Uri.UriSchemeHttp && parsed.Scheme != Uri.UriSchemeHttps)) + { + return; + } + + try + { + references.Add(new AdvisoryReference(parsed.ToString(), kind, sourceTag, null, provenance)); + } + catch (ArgumentException) + { + // Ignore invalid references while leaving traceability via diagnostics elsewhere. + } + } + + private static string? BuildCatalogSearchUrl(string cveId) + { + if (string.IsNullOrWhiteSpace(cveId)) + { + return null; + } + + var builder = new StringBuilder("https://www.cisa.gov/known-exploited-vulnerabilities-catalog?search="); + builder.Append(Uri.EscapeDataString(cveId)); + return builder.ToString(); + } + + private static IEnumerable BuildAffectedPackages( + KevVulnerabilityDto entry, + KevCatalogDto catalog, + string sourceName, + AdvisoryProvenance mappingProvenance, + DateTimeOffset? published, + DateTimeOffset? dueDate) + { + var identifier = BuildIdentifier(entry) ?? entry.CveId ?? "kev"; + var rangeExtensions = new Dictionary(StringComparer.OrdinalIgnoreCase); + + void TryAddExtension(string key, string? value, int maxLength = 512) + { + if (string.IsNullOrWhiteSpace(value)) + { + return; + } + + var trimmed = value.Trim(); + if (trimmed.Length > maxLength) + { + trimmed = trimmed[..maxLength].Trim(); + } + + if (trimmed.Length > 0) + { + rangeExtensions[key] = trimmed; + } + } + + TryAddExtension("kev.vendorProject", entry.VendorProject, 256); + TryAddExtension("kev.product", entry.Product, 256); + TryAddExtension("kev.requiredAction", entry.RequiredAction); + TryAddExtension("kev.knownRansomwareCampaignUse", entry.KnownRansomwareCampaignUse, 64); + TryAddExtension("kev.notes", entry.Notes); + TryAddExtension("kev.catalogVersion", catalog.CatalogVersion, 64); + + if (catalog.DateReleased.HasValue) + { + TryAddExtension("kev.catalogReleased", catalog.DateReleased.Value.ToString("O", CultureInfo.InvariantCulture)); + } + + if (published.HasValue) + { + TryAddExtension("kev.dateAdded", published.Value.ToString("yyyy-MM-dd", CultureInfo.InvariantCulture)); + } + + if (dueDate.HasValue) + { + TryAddExtension("kev.dueDate", dueDate.Value.ToString("yyyy-MM-dd", CultureInfo.InvariantCulture)); + } + + if (entry.Cwes is { Count: > 0 }) + { + TryAddExtension("kev.cwe", string.Join(",", entry.Cwes.Where(static cwe => !string.IsNullOrWhiteSpace(cwe)).OrderBy(static cwe => cwe, StringComparer.Ordinal))); + } + + if (rangeExtensions.Count == 0) + { + return Array.Empty(); + } + + var rangeProvenance = new AdvisoryProvenance(sourceName, "kev-range", identifier, mappingProvenance.RecordedAt); + var range = new AffectedVersionRange( + rangeKind: AffectedPackageTypes.Vendor, + introducedVersion: null, + fixedVersion: null, + lastAffectedVersion: null, + rangeExpression: null, + provenance: rangeProvenance, + primitives: new RangePrimitives(null, null, null, rangeExtensions)); + + var affectedPackage = new AffectedPackage( + AffectedPackageTypes.Vendor, + identifier, + platform: null, + versionRanges: new[] { range }, + statuses: Array.Empty(), + provenance: new[] { mappingProvenance }); + + return new[] { affectedPackage }; + } + + private static string? BuildIdentifier(KevVulnerabilityDto entry) + { + var vendor = Normalize(entry.VendorProject); + var product = Normalize(entry.Product); + + if (!string.IsNullOrEmpty(vendor) && !string.IsNullOrEmpty(product)) + { + return $"{vendor}::{product}"; + } + + return vendor ?? product; + } + + private static IEnumerable ExtractUrls(string? notes) + { + if (string.IsNullOrWhiteSpace(notes)) + { + return Array.Empty(); + } + + var tokens = notes.Split(new[] { ';', ',', ' ', '\r', '\n', '\t' }, StringSplitOptions.RemoveEmptyEntries); + var results = new List(); + + foreach (var token in tokens) + { + var trimmed = token.Trim().TrimEnd('.', ')', ';', ','); + if (trimmed.Length == 0) + { + continue; + } + + if (Uri.TryCreate(trimmed, UriKind.Absolute, out var uri) + && (uri.Scheme == Uri.UriSchemeHttp || uri.Scheme == Uri.UriSchemeHttps)) + { + results.Add(uri.ToString()); + } + } + + return results.Count == 0 + ? Array.Empty() + : results.Distinct(StringComparer.OrdinalIgnoreCase).OrderBy(static value => value, StringComparer.Ordinal).ToArray(); + } + + private static string? Normalize(string? value) + { + if (string.IsNullOrWhiteSpace(value)) + { + return null; + } + + var trimmed = value.Trim(); + return trimmed.Length == 0 ? null : trimmed; + } + + private static DateTimeOffset? ParseDate(string? value) + { + if (string.IsNullOrWhiteSpace(value)) + { + return null; + } + + if (DateTimeOffset.TryParse(value, CultureInfo.InvariantCulture, DateTimeStyles.AssumeUniversal | DateTimeStyles.AdjustToUniversal, out var parsed)) + { + return parsed.ToUniversalTime(); + } + + if (DateTime.TryParse(value, CultureInfo.InvariantCulture, DateTimeStyles.AssumeUniversal | DateTimeStyles.AdjustToUniversal, out var date)) + { + return new DateTimeOffset(DateTime.SpecifyKind(date, DateTimeKind.Utc)); + } + + return null; + } +} diff --git a/src/StellaOps.Feedser.Source.Kev/Internal/KevSchemaProvider.cs b/src/StellaOps.Feedser.Source.Kev/Internal/KevSchemaProvider.cs new file mode 100644 index 00000000..53f1287b --- /dev/null +++ b/src/StellaOps.Feedser.Source.Kev/Internal/KevSchemaProvider.cs @@ -0,0 +1,25 @@ +using System.IO; +using System.Reflection; +using System.Threading; +using Json.Schema; + +namespace StellaOps.Feedser.Source.Kev.Internal; + +internal static class KevSchemaProvider +{ + private const string ResourceName = "StellaOps.Feedser.Source.Kev.Schemas.kev-catalog.schema.json"; + + private static readonly Lazy CachedSchema = new(LoadSchema, LazyThreadSafetyMode.ExecutionAndPublication); + + public static JsonSchema Schema => CachedSchema.Value; + + private static JsonSchema LoadSchema() + { + var assembly = typeof(KevSchemaProvider).GetTypeInfo().Assembly; + using var stream = assembly.GetManifestResourceStream(ResourceName) + ?? throw new InvalidOperationException($"Embedded schema '{ResourceName}' was not found."); + using var reader = new StreamReader(stream); + var schemaJson = reader.ReadToEnd(); + return JsonSchema.FromText(schemaJson); + } +} diff --git a/src/StellaOps.Feedser.Source.Kev/Jobs.cs b/src/StellaOps.Feedser.Source.Kev/Jobs.cs new file mode 100644 index 00000000..f9323c9d --- /dev/null +++ b/src/StellaOps.Feedser.Source.Kev/Jobs.cs @@ -0,0 +1,46 @@ +using System; +using System.Threading; +using System.Threading.Tasks; +using StellaOps.Feedser.Core.Jobs; + +namespace StellaOps.Feedser.Source.Kev; + +internal static class KevJobKinds +{ + public const string Fetch = "source:kev:fetch"; + public const string Parse = "source:kev:parse"; + public const string Map = "source:kev:map"; +} + +internal sealed class KevFetchJob : IJob +{ + private readonly KevConnector _connector; + + public KevFetchJob(KevConnector connector) + => _connector = connector ?? throw new ArgumentNullException(nameof(connector)); + + public Task ExecuteAsync(JobExecutionContext context, CancellationToken cancellationToken) + => _connector.FetchAsync(context.Services, cancellationToken); +} + +internal sealed class KevParseJob : IJob +{ + private readonly KevConnector _connector; + + public KevParseJob(KevConnector connector) + => _connector = connector ?? throw new ArgumentNullException(nameof(connector)); + + public Task ExecuteAsync(JobExecutionContext context, CancellationToken cancellationToken) + => _connector.ParseAsync(context.Services, cancellationToken); +} + +internal sealed class KevMapJob : IJob +{ + private readonly KevConnector _connector; + + public KevMapJob(KevConnector connector) + => _connector = connector ?? throw new ArgumentNullException(nameof(connector)); + + public Task ExecuteAsync(JobExecutionContext context, CancellationToken cancellationToken) + => _connector.MapAsync(context.Services, cancellationToken); +} diff --git a/src/StellaOps.Feedser.Source.Kev/KevConnector.cs b/src/StellaOps.Feedser.Source.Kev/KevConnector.cs new file mode 100644 index 00000000..6a01a660 --- /dev/null +++ b/src/StellaOps.Feedser.Source.Kev/KevConnector.cs @@ -0,0 +1,323 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text.Json; +using System.Text.Json.Serialization; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Options; +using MongoDB.Bson; +using StellaOps.Feedser.Models; +using StellaOps.Feedser.Source.Common; +using StellaOps.Feedser.Source.Common.Fetch; +using StellaOps.Feedser.Source.Kev.Configuration; +using StellaOps.Feedser.Source.Kev.Internal; +using StellaOps.Feedser.Storage.Mongo; +using StellaOps.Feedser.Storage.Mongo.Advisories; +using StellaOps.Feedser.Storage.Mongo.Documents; +using StellaOps.Feedser.Storage.Mongo.Dtos; +using StellaOps.Plugin; + +namespace StellaOps.Feedser.Source.Kev; + +public sealed class KevConnector : IFeedConnector +{ + private static readonly JsonSerializerOptions SerializerOptions = new(JsonSerializerDefaults.Web) + { + PropertyNameCaseInsensitive = true, + DefaultIgnoreCondition = JsonIgnoreCondition.WhenWritingNull, + }; + + private const string SchemaVersion = "kev.catalog.v1"; + + private readonly SourceFetchService _fetchService; + private readonly RawDocumentStorage _rawDocumentStorage; + private readonly IDocumentStore _documentStore; + private readonly IDtoStore _dtoStore; + private readonly IAdvisoryStore _advisoryStore; + private readonly ISourceStateRepository _stateRepository; + private readonly KevOptions _options; + private readonly TimeProvider _timeProvider; + private readonly ILogger _logger; + private readonly KevDiagnostics _diagnostics; + + public KevConnector( + SourceFetchService fetchService, + RawDocumentStorage rawDocumentStorage, + IDocumentStore documentStore, + IDtoStore dtoStore, + IAdvisoryStore advisoryStore, + ISourceStateRepository stateRepository, + IOptions options, + KevDiagnostics diagnostics, + TimeProvider? timeProvider, + ILogger logger) + { + _fetchService = fetchService ?? throw new ArgumentNullException(nameof(fetchService)); + _rawDocumentStorage = rawDocumentStorage ?? throw new ArgumentNullException(nameof(rawDocumentStorage)); + _documentStore = documentStore ?? throw new ArgumentNullException(nameof(documentStore)); + _dtoStore = dtoStore ?? throw new ArgumentNullException(nameof(dtoStore)); + _advisoryStore = advisoryStore ?? throw new ArgumentNullException(nameof(advisoryStore)); + _stateRepository = stateRepository ?? throw new ArgumentNullException(nameof(stateRepository)); + _options = options?.Value ?? throw new ArgumentNullException(nameof(options)); + _options.Validate(); + _diagnostics = diagnostics ?? throw new ArgumentNullException(nameof(diagnostics)); + _timeProvider = timeProvider ?? TimeProvider.System; + _logger = logger ?? throw new ArgumentNullException(nameof(logger)); + } + + public string SourceName => KevConnectorPlugin.SourceName; + + public async Task FetchAsync(IServiceProvider services, CancellationToken cancellationToken) + { + ArgumentNullException.ThrowIfNull(services); + + var cursor = await GetCursorAsync(cancellationToken).ConfigureAwait(false); + var now = _timeProvider.GetUtcNow(); + + try + { + var existing = await _documentStore.FindBySourceAndUriAsync(SourceName, _options.FeedUri.ToString(), cancellationToken).ConfigureAwait(false); + + var request = new SourceFetchRequest( + KevOptions.HttpClientName, + SourceName, + _options.FeedUri) + { + Metadata = new Dictionary(StringComparer.Ordinal) + { + ["kev.cursor.catalogVersion"] = cursor.CatalogVersion ?? string.Empty, + ["kev.cursor.catalogReleased"] = cursor.CatalogReleased?.ToString("O") ?? string.Empty, + }, + ETag = existing?.Etag, + LastModified = existing?.LastModified, + TimeoutOverride = _options.RequestTimeout, + AcceptHeaders = new[] { "application/json", "text/json" }, + }; + + var result = await _fetchService.FetchAsync(request, cancellationToken).ConfigureAwait(false); + if (result.IsNotModified) + { + await UpdateCursorAsync(cursor, cancellationToken).ConfigureAwait(false); + return; + } + + if (!result.IsSuccess || result.Document is null) + { + await _stateRepository.MarkFailureAsync(SourceName, now, TimeSpan.FromMinutes(5), "KEV feed returned no content.", cancellationToken).ConfigureAwait(false); + return; + } + + var pendingDocuments = cursor.PendingDocuments.ToHashSet(); + pendingDocuments.Add(result.Document.Id); + + var updatedCursor = cursor + .WithPendingDocuments(pendingDocuments) + .WithPendingMappings(cursor.PendingMappings); + + await UpdateCursorAsync(updatedCursor, cancellationToken).ConfigureAwait(false); + } + catch (Exception ex) + { + _logger.LogError(ex, "KEV fetch failed for {Uri}", _options.FeedUri); + await _stateRepository.MarkFailureAsync(SourceName, now, TimeSpan.FromMinutes(5), ex.Message, cancellationToken).ConfigureAwait(false); + throw; + } + } + + public async Task ParseAsync(IServiceProvider services, CancellationToken cancellationToken) + { + ArgumentNullException.ThrowIfNull(services); + + var cursor = await GetCursorAsync(cancellationToken).ConfigureAwait(false); + if (cursor.PendingDocuments.Count == 0) + { + return; + } + + var remainingDocuments = cursor.PendingDocuments.ToList(); + var pendingMappings = cursor.PendingMappings.ToHashSet(); + var latestCatalogVersion = cursor.CatalogVersion; + var latestCatalogReleased = cursor.CatalogReleased; + + foreach (var documentId in cursor.PendingDocuments) + { + cancellationToken.ThrowIfCancellationRequested(); + + var document = await _documentStore.FindAsync(documentId, cancellationToken).ConfigureAwait(false); + if (document is null) + { + remainingDocuments.Remove(documentId); + pendingMappings.Remove(documentId); + continue; + } + + if (!document.GridFsId.HasValue) + { + _logger.LogWarning("KEV document {DocumentId} missing GridFS payload", document.Id); + await _documentStore.UpdateStatusAsync(document.Id, DocumentStatuses.Failed, cancellationToken).ConfigureAwait(false); + remainingDocuments.Remove(documentId); + pendingMappings.Remove(documentId); + continue; + } + + KevCatalogDto? catalog; + try + { + var rawBytes = await _rawDocumentStorage.DownloadAsync(document.GridFsId.Value, cancellationToken).ConfigureAwait(false); + catalog = JsonSerializer.Deserialize(rawBytes, SerializerOptions); + } + catch (Exception ex) + { + _logger.LogError(ex, "KEV parse failed for document {DocumentId}", document.Id); + await _documentStore.UpdateStatusAsync(document.Id, DocumentStatuses.Failed, cancellationToken).ConfigureAwait(false); + remainingDocuments.Remove(documentId); + pendingMappings.Remove(documentId); + continue; + } + + if (catalog is null) + { + _logger.LogWarning("KEV catalog payload was empty for document {DocumentId}", document.Id); + await _documentStore.UpdateStatusAsync(document.Id, DocumentStatuses.Failed, cancellationToken).ConfigureAwait(false); + remainingDocuments.Remove(documentId); + pendingMappings.Remove(documentId); + continue; + } + + try + { + var payloadJson = JsonSerializer.Serialize(catalog, SerializerOptions); + var payload = BsonDocument.Parse(payloadJson); + var entryCount = catalog.Vulnerabilities?.Count ?? 0; + var released = catalog.DateReleased?.ToUniversalTime(); + + _logger.LogInformation( + "Parsed KEV catalog document {DocumentId} (version={CatalogVersion}, released={Released}, entries={EntryCount})", + document.Id, + catalog.CatalogVersion ?? "(unknown)", + released, + entryCount); + _diagnostics.CatalogParsed(catalog.CatalogVersion, entryCount); + + var dtoRecord = new DtoRecord( + Guid.NewGuid(), + document.Id, + SourceName, + SchemaVersion, + payload, + _timeProvider.GetUtcNow()); + + await _dtoStore.UpsertAsync(dtoRecord, cancellationToken).ConfigureAwait(false); + await _documentStore.UpdateStatusAsync(document.Id, DocumentStatuses.PendingMap, cancellationToken).ConfigureAwait(false); + + remainingDocuments.Remove(documentId); + pendingMappings.Add(document.Id); + + latestCatalogVersion = catalog.CatalogVersion ?? latestCatalogVersion; + latestCatalogReleased = catalog.DateReleased ?? latestCatalogReleased; + } + catch (Exception ex) + { + _logger.LogError(ex, "KEV DTO persistence failed for document {DocumentId}", document.Id); + await _documentStore.UpdateStatusAsync(document.Id, DocumentStatuses.Failed, cancellationToken).ConfigureAwait(false); + remainingDocuments.Remove(documentId); + pendingMappings.Remove(documentId); + } + } + + var updatedCursor = cursor + .WithPendingDocuments(remainingDocuments) + .WithPendingMappings(pendingMappings) + .WithCatalogMetadata(latestCatalogVersion, latestCatalogReleased); + + await UpdateCursorAsync(updatedCursor, cancellationToken).ConfigureAwait(false); + } + + public async Task MapAsync(IServiceProvider services, CancellationToken cancellationToken) + { + ArgumentNullException.ThrowIfNull(services); + + var cursor = await GetCursorAsync(cancellationToken).ConfigureAwait(false); + if (cursor.PendingMappings.Count == 0) + { + return; + } + + var pendingMappings = cursor.PendingMappings.ToHashSet(); + + foreach (var documentId in cursor.PendingMappings) + { + cancellationToken.ThrowIfCancellationRequested(); + + var dtoRecord = await _dtoStore.FindByDocumentIdAsync(documentId, cancellationToken).ConfigureAwait(false); + var document = await _documentStore.FindAsync(documentId, cancellationToken).ConfigureAwait(false); + + if (dtoRecord is null || document is null) + { + pendingMappings.Remove(documentId); + continue; + } + + KevCatalogDto? catalog; + try + { + var dtoJson = dtoRecord.Payload.ToJson(new MongoDB.Bson.IO.JsonWriterSettings + { + OutputMode = MongoDB.Bson.IO.JsonOutputMode.RelaxedExtendedJson, + }); + + catalog = JsonSerializer.Deserialize(dtoJson, SerializerOptions); + } + catch (Exception ex) + { + _logger.LogError(ex, "KEV mapping: failed to deserialize DTO for document {DocumentId}", document.Id); + await _documentStore.UpdateStatusAsync(document.Id, DocumentStatuses.Failed, cancellationToken).ConfigureAwait(false); + pendingMappings.Remove(documentId); + continue; + } + + if (catalog is null) + { + _logger.LogWarning("KEV mapping: DTO payload was empty for document {DocumentId}", document.Id); + await _documentStore.UpdateStatusAsync(document.Id, DocumentStatuses.Failed, cancellationToken).ConfigureAwait(false); + pendingMappings.Remove(documentId); + continue; + } + + var feedUri = TryParseUri(document.Uri) ?? _options.FeedUri; + var advisories = KevMapper.Map(catalog, SourceName, feedUri, document.FetchedAt, dtoRecord.ValidatedAt); + _logger.LogInformation( + "Mapped {AdvisoryCount} KEV advisories from catalog version {CatalogVersion}", + advisories.Count, + catalog.CatalogVersion ?? "(unknown)"); + _diagnostics.AdvisoriesMapped(catalog.CatalogVersion, advisories.Count); + + foreach (var advisory in advisories) + { + await _advisoryStore.UpsertAsync(advisory, cancellationToken).ConfigureAwait(false); + } + + await _documentStore.UpdateStatusAsync(document.Id, DocumentStatuses.Mapped, cancellationToken).ConfigureAwait(false); + pendingMappings.Remove(documentId); + } + + var updatedCursor = cursor.WithPendingMappings(pendingMappings); + await UpdateCursorAsync(updatedCursor, cancellationToken).ConfigureAwait(false); + } + + private async Task GetCursorAsync(CancellationToken cancellationToken) + { + var state = await _stateRepository.TryGetAsync(SourceName, cancellationToken).ConfigureAwait(false); + return state is null ? KevCursor.Empty : KevCursor.FromBson(state.Cursor); + } + + private Task UpdateCursorAsync(KevCursor cursor, CancellationToken cancellationToken) + { + return _stateRepository.UpdateCursorAsync(SourceName, cursor.ToBsonDocument(), _timeProvider.GetUtcNow(), cancellationToken); + } + + private static Uri? TryParseUri(string? value) + => Uri.TryCreate(value, UriKind.Absolute, out var uri) ? uri : null; +} diff --git a/src/StellaOps.Feedser.Source.Kev/KevConnectorPlugin.cs b/src/StellaOps.Feedser.Source.Kev/KevConnectorPlugin.cs new file mode 100644 index 00000000..0905f025 --- /dev/null +++ b/src/StellaOps.Feedser.Source.Kev/KevConnectorPlugin.cs @@ -0,0 +1,19 @@ +using Microsoft.Extensions.DependencyInjection; +using StellaOps.Plugin; + +namespace StellaOps.Feedser.Source.Kev; + +public sealed class KevConnectorPlugin : IConnectorPlugin +{ + public const string SourceName = "kev"; + + public string Name => SourceName; + + public bool IsAvailable(IServiceProvider services) => services is not null; + + public IFeedConnector Create(IServiceProvider services) + { + ArgumentNullException.ThrowIfNull(services); + return ActivatorUtilities.CreateInstance(services); + } +} diff --git a/src/StellaOps.Feedser.Source.Kev/KevDependencyInjectionRoutine.cs b/src/StellaOps.Feedser.Source.Kev/KevDependencyInjectionRoutine.cs new file mode 100644 index 00000000..27e4274e --- /dev/null +++ b/src/StellaOps.Feedser.Source.Kev/KevDependencyInjectionRoutine.cs @@ -0,0 +1,54 @@ +using System; +using Microsoft.Extensions.Configuration; +using Microsoft.Extensions.DependencyInjection; +using StellaOps.DependencyInjection; +using StellaOps.Feedser.Core.Jobs; +using StellaOps.Feedser.Source.Kev.Configuration; + +namespace StellaOps.Feedser.Source.Kev; + +public sealed class KevDependencyInjectionRoutine : IDependencyInjectionRoutine +{ + private const string ConfigurationSection = "feedser:sources:kev"; + + public IServiceCollection Register(IServiceCollection services, IConfiguration configuration) + { + ArgumentNullException.ThrowIfNull(services); + ArgumentNullException.ThrowIfNull(configuration); + + services.AddKevConnector(options => + { + configuration.GetSection(ConfigurationSection).Bind(options); + options.Validate(); + }); + + services.AddTransient(); + services.AddTransient(); + services.AddTransient(); + + services.PostConfigure(options => + { + EnsureJob(options, KevJobKinds.Fetch, typeof(KevFetchJob)); + EnsureJob(options, KevJobKinds.Parse, typeof(KevParseJob)); + EnsureJob(options, KevJobKinds.Map, typeof(KevMapJob)); + }); + + return services; + } + + private static void EnsureJob(JobSchedulerOptions options, string kind, Type jobType) + { + if (options.Definitions.ContainsKey(kind)) + { + return; + } + + options.Definitions[kind] = new JobDefinition( + kind, + jobType, + options.DefaultTimeout, + options.DefaultLeaseDuration, + CronExpression: null, + Enabled: true); + } +} diff --git a/src/StellaOps.Feedser.Source.Kev/KevServiceCollectionExtensions.cs b/src/StellaOps.Feedser.Source.Kev/KevServiceCollectionExtensions.cs new file mode 100644 index 00000000..eeb5cd95 --- /dev/null +++ b/src/StellaOps.Feedser.Source.Kev/KevServiceCollectionExtensions.cs @@ -0,0 +1,37 @@ +using System; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.DependencyInjection.Extensions; +using Microsoft.Extensions.Options; +using StellaOps.Feedser.Source.Common.Http; +using StellaOps.Feedser.Source.Kev.Configuration; + +namespace StellaOps.Feedser.Source.Kev; + +public static class KevServiceCollectionExtensions +{ + public static IServiceCollection AddKevConnector(this IServiceCollection services, Action configure) + { + ArgumentNullException.ThrowIfNull(services); + ArgumentNullException.ThrowIfNull(configure); + + services.AddOptions() + .Configure(configure) + .PostConfigure(static options => options.Validate()); + + services.AddSourceHttpClient(KevOptions.HttpClientName, (provider, clientOptions) => + { + var opts = provider.GetRequiredService>().Value; + clientOptions.BaseAddress = opts.FeedUri; + clientOptions.Timeout = opts.RequestTimeout; + clientOptions.UserAgent = "StellaOps.Feedser.Kev/1.0"; + clientOptions.AllowedHosts.Clear(); + clientOptions.AllowedHosts.Add(opts.FeedUri.Host); + clientOptions.DefaultRequestHeaders["Accept"] = "application/json"; + }); + + services.TryAddSingleton(); + services.AddTransient(); + + return services; + } +} diff --git a/src/StellaOps.Feedser.Source.Kev/Schemas/kev-catalog.schema.json b/src/StellaOps.Feedser.Source.Kev/Schemas/kev-catalog.schema.json new file mode 100644 index 00000000..53860c77 --- /dev/null +++ b/src/StellaOps.Feedser.Source.Kev/Schemas/kev-catalog.schema.json @@ -0,0 +1,80 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "title": "CISA Known Exploited Vulnerabilities catalog", + "type": "object", + "required": [ + "catalogVersion", + "dateReleased", + "count", + "vulnerabilities" + ], + "properties": { + "title": { + "type": "string" + }, + "catalogVersion": { + "type": "string", + "minLength": 1 + }, + "dateReleased": { + "type": "string", + "format": "date-time" + }, + "count": { + "type": "integer", + "minimum": 0 + }, + "vulnerabilities": { + "type": "array", + "items": { + "type": "object", + "required": [ + "cveID", + "vendorProject", + "product" + ], + "properties": { + "cveID": { + "type": "string", + "pattern": "^CVE-\\d{4}-\\d{4,}$" + }, + "vendorProject": { + "type": "string" + }, + "product": { + "type": "string" + }, + "vulnerabilityName": { + "type": "string" + }, + "dateAdded": { + "type": "string" + }, + "shortDescription": { + "type": "string" + }, + "requiredAction": { + "type": "string" + }, + "dueDate": { + "type": "string" + }, + "knownRansomwareCampaignUse": { + "type": "string" + }, + "notes": { + "type": "string" + }, + "cwes": { + "type": "array", + "items": { + "type": "string" + } + } + }, + "additionalProperties": true + } + } + }, + "additionalProperties": true +} diff --git a/src/StellaOps.Feedser.Source.Kev/StellaOps.Feedser.Source.Kev.csproj b/src/StellaOps.Feedser.Source.Kev/StellaOps.Feedser.Source.Kev.csproj new file mode 100644 index 00000000..f9ee61f0 --- /dev/null +++ b/src/StellaOps.Feedser.Source.Kev/StellaOps.Feedser.Source.Kev.csproj @@ -0,0 +1,23 @@ + + + + net10.0 + enable + enable + + + + + + + + + + + + + <_Parameter1>StellaOps.Feedser.Source.Kev.Tests + + + + diff --git a/src/StellaOps.Feedser.Source.Kev/TASKS.md b/src/StellaOps.Feedser.Source.Kev/TASKS.md new file mode 100644 index 00000000..51b55b0c --- /dev/null +++ b/src/StellaOps.Feedser.Source.Kev/TASKS.md @@ -0,0 +1,11 @@ +# TASKS +| Task | Owner(s) | Depends on | Notes | +|---|---|---|---| +|Review KEV JSON schema & cadence|BE-Conn-KEV|Research|**DONE** – Feed defaults lock to the public JSON catalog; AGENTS notes call out daily cadence and allowlist requirements.| +|Fetch & cursor implementation|BE-Conn-KEV|Source.Common, Storage.Mongo|**DONE** – SourceFetchService drives ETag/Last-Modified aware fetches with SourceState cursor tracking documents + catalog metadata.| +|DTO/parser implementation|BE-Conn-KEV|Source.Common|**DONE** – `KevCatalogDto`/`KevVulnerabilityDto` deserialize payloads with logging for catalog version/releases before DTO persistence.| +|Canonical mapping & range primitives|BE-Conn-KEV|Models|**DONE** – Mapper produces vendor RangePrimitives (due dates, CWE list, ransomware flag, catalog metadata) and deduplicated references.| +|Deterministic fixtures/tests|QA|Testing|**DONE** – End-to-end fetch→parse→map test with canned catalog + snapshot (`UPDATE_KEV_FIXTURES=1`) guards determinism.| +|Telemetry & docs|DevEx|Docs|**DONE** – Connector emits structured logs + meters for catalog entries/advisories and AGENTS docs cover cadence/allowlist guidance.| +|Schema validation & anomaly surfacing|BE-Conn-KEV, QA|Source.Common|**TODO** – Introduce JSON schema validation for catalog payloads, quarantine malformed entries, and add unit tests covering null/empty fields plus failure logging.| +|Metrics export wiring|DevOps, DevEx|Observability|**TODO** – Plumb `kev.*` counters into shared OTEL pipeline, document metric names/labels, and update dashboards/alerts for sustained zero-entry releases or spikes.| diff --git a/src/StellaOps.Feedser.Source.Kisa/AGENTS.md b/src/StellaOps.Feedser.Source.Kisa/AGENTS.md new file mode 100644 index 00000000..8efa4065 --- /dev/null +++ b/src/StellaOps.Feedser.Source.Kisa/AGENTS.md @@ -0,0 +1,38 @@ +# AGENTS +## Role +Deliver the KISA (Korea Internet & Security Agency) advisory connector to ingest Korean vulnerability alerts for Feedser’s regional coverage. + +## Scope +- Identify KISA’s advisory feeds (RSS/Atom, JSON, HTML) and determine localisation requirements (Korean language parsing). +- Implement fetch/cursor logic with retry/backoff, handling authentication if required. +- Parse advisory content to extract summary, affected vendors/products, mitigation steps, CVEs, references. +- Map advisories into canonical `Advisory` records with aliases, references, affected packages, and range primitives (including vendor/language metadata). +- Provide deterministic fixtures and regression tests. + +## Participants +- `Source.Common` (HTTP/fetch utilities, DTO storage). +- `Storage.Mongo` (raw/document/DTO/advisory stores, source state). +- `Feedser.Models` (canonical data structures). +- `Feedser.Testing` (integration fixtures and snapshots). + +## Interfaces & Contracts +- Job kinds: `kisa:fetch`, `kisa:parse`, `kisa:map`. +- Persist upstream caching metadata (e.g., ETag/Last-Modified) when available. +- Alias set should include KISA advisory identifiers and CVE IDs. + +## In/Out of scope +In scope: +- Advisory ingestion, translation/normalisation, range primitives. + +Out of scope: +- Automated Korean↔English translations beyond summary normalization (unless required for canonical fields). + +## Observability & Security Expectations +- Log fetch and mapping metrics; record failures with backoff. +- Sanitise HTML, removing scripts/styles. +- Handle character encoding (UTF-8/Korean) correctly. + +## Tests +- Add `StellaOps.Feedser.Source.Kisa.Tests` covering fetch/parse/map with Korean-language fixtures. +- Snapshot canonical advisories; support fixture regeneration via env flag. +- Ensure deterministic ordering/time normalisation. diff --git a/src/StellaOps.Feedser.Source.Kisa/Class1.cs b/src/StellaOps.Feedser.Source.Kisa/Class1.cs new file mode 100644 index 00000000..f8db6a87 --- /dev/null +++ b/src/StellaOps.Feedser.Source.Kisa/Class1.cs @@ -0,0 +1,29 @@ +using System; +using System.Threading; +using System.Threading.Tasks; +using StellaOps.Plugin; + +namespace StellaOps.Feedser.Source.Kisa; + +public sealed class KisaConnectorPlugin : IConnectorPlugin +{ + public string Name => "kisa"; + + public bool IsAvailable(IServiceProvider services) => true; + + public IFeedConnector Create(IServiceProvider services) => new StubConnector(Name); + + private sealed class StubConnector : IFeedConnector + { + public StubConnector(string sourceName) => SourceName = sourceName; + + public string SourceName { get; } + + public Task FetchAsync(IServiceProvider services, CancellationToken cancellationToken) => Task.CompletedTask; + + public Task ParseAsync(IServiceProvider services, CancellationToken cancellationToken) => Task.CompletedTask; + + public Task MapAsync(IServiceProvider services, CancellationToken cancellationToken) => Task.CompletedTask; + } +} + diff --git a/src/StellaOps.Feedser.Source.Kisa/StellaOps.Feedser.Source.Kisa.csproj b/src/StellaOps.Feedser.Source.Kisa/StellaOps.Feedser.Source.Kisa.csproj new file mode 100644 index 00000000..f7f2c154 --- /dev/null +++ b/src/StellaOps.Feedser.Source.Kisa/StellaOps.Feedser.Source.Kisa.csproj @@ -0,0 +1,16 @@ + + + + net10.0 + enable + enable + + + + + + + + + + diff --git a/src/StellaOps.Feedser.Source.Kisa/TASKS.md b/src/StellaOps.Feedser.Source.Kisa/TASKS.md new file mode 100644 index 00000000..3e2289c9 --- /dev/null +++ b/src/StellaOps.Feedser.Source.Kisa/TASKS.md @@ -0,0 +1,10 @@ +# TASKS +| Task | Owner(s) | Depends on | Notes | +|---|---|---|---| +|FEEDCONN-KISA-02-001 Research KISA advisory feeds|BE-Conn-KISA|Research|**DOING** – 2025-10-11: Identified KNVD portal RSS endpoint (`https://knvd.krcert.or.kr/kr/reporter/rss.do`) but access currently returns HTML front-door; need session/header strategy, schema samples, and localisation assessment.| +|FEEDCONN-KISA-02-002 Fetch pipeline & source state|BE-Conn-KISA|Source.Common, Storage.Mongo|**TODO** – Configure HTTP client (encoding-aware), implement fetch job with cursor persistence/backoff.| +|FEEDCONN-KISA-02-003 Parser & DTO implementation|BE-Conn-KISA|Source.Common|**TODO** – Create DTOs handling Korean text, extract summary, CVEs, references, vendor/product data.| +|FEEDCONN-KISA-02-004 Canonical mapping & range primitives|BE-Conn-KISA|Models|**TODO** – Map advisories to canonical records with aliases, references, and vendor/language range primitives. Follow localization guidance while aligning SemVer outputs per `../StellaOps.Feedser.Merge/RANGE_PRIMITIVES_COORDINATION.md`.| +|FEEDCONN-KISA-02-005 Deterministic fixtures & tests|QA|Testing|**TODO** – Add regression tests with Korean-language fixtures; support `UPDATE_KISA_FIXTURES=1`.| +|FEEDCONN-KISA-02-006 Telemetry & documentation|DevEx|Docs|**TODO** – Add logging/metrics and document connector configuration once implemented.| +|FEEDCONN-KISA-02-007 RSS contract & localisation brief|BE-Conn-KISA|Research|**TODO** – Capture concrete RSS feed URLs, required headers, character encodings, and outline translation strategy for advisory titles/bodies.| diff --git a/src/StellaOps.Feedser.Source.Nvd.Tests/Nvd/Fixtures/nvd-invalid-schema.json b/src/StellaOps.Feedser.Source.Nvd.Tests/Nvd/Fixtures/nvd-invalid-schema.json new file mode 100644 index 00000000..02b611d5 --- /dev/null +++ b/src/StellaOps.Feedser.Source.Nvd.Tests/Nvd/Fixtures/nvd-invalid-schema.json @@ -0,0 +1,6 @@ +{ + "resultsPerPage": 1, + "startIndex": 0, + "totalResults": 1, + "vulnerabilities": "this-should-be-an-array" +} diff --git a/src/StellaOps.Feedser.Source.Nvd.Tests/Nvd/Fixtures/nvd-multipage-1.json b/src/StellaOps.Feedser.Source.Nvd.Tests/Nvd/Fixtures/nvd-multipage-1.json new file mode 100644 index 00000000..e6a45070 --- /dev/null +++ b/src/StellaOps.Feedser.Source.Nvd.Tests/Nvd/Fixtures/nvd-multipage-1.json @@ -0,0 +1,69 @@ +{ + "resultsPerPage": 2, + "startIndex": 0, + "totalResults": 5, + "vulnerabilities": [ + { + "cve": { + "id": "CVE-2024-1000", + "sourceIdentifier": "nvd@nist.gov", + "published": "2024-02-01T10:00:00Z", + "lastModified": "2024-02-02T10:00:00Z", + "descriptions": [ + { "lang": "en", "value": "Multipage vulnerability one." } + ], + "metrics": { + "cvssMetricV31": [ + { + "cvssData": { + "vectorString": "CVSS:3.1/AV:N/AC:L/PR:N/UI:N/S:U/C:H/I:H/A:H", + "baseScore": 9.8, + "baseSeverity": "CRITICAL" + } + } + ] + }, + "configurations": { + "nodes": [ + { + "cpeMatch": [ + { "vulnerable": true, "criteria": "cpe:2.3:a:example:product_a:1.0:*:*:*:*:*:*:*" } + ] + } + ] + } + } + }, + { + "cve": { + "id": "CVE-2024-1001", + "sourceIdentifier": "nvd@nist.gov", + "published": "2024-02-01T11:00:00Z", + "lastModified": "2024-02-02T11:00:00Z", + "descriptions": [ + { "lang": "en", "value": "Multipage vulnerability two." } + ], + "metrics": { + "cvssMetricV31": [ + { + "cvssData": { + "vectorString": "CVSS:3.1/AV:P/AC:L/PR:L/UI:R/S:U/C:L/I:L/A:L", + "baseScore": 5.1, + "baseSeverity": "MEDIUM" + } + } + ] + }, + "configurations": { + "nodes": [ + { + "cpeMatch": [ + { "vulnerable": true, "criteria": "cpe:2.3:a:example:product_b:2.0:*:*:*:*:*:*:*" } + ] + } + ] + } + } + } + ] +} diff --git a/src/StellaOps.Feedser.Source.Nvd.Tests/Nvd/Fixtures/nvd-multipage-2.json b/src/StellaOps.Feedser.Source.Nvd.Tests/Nvd/Fixtures/nvd-multipage-2.json new file mode 100644 index 00000000..0270b45d --- /dev/null +++ b/src/StellaOps.Feedser.Source.Nvd.Tests/Nvd/Fixtures/nvd-multipage-2.json @@ -0,0 +1,69 @@ +{ + "resultsPerPage": 2, + "startIndex": 2, + "totalResults": 5, + "vulnerabilities": [ + { + "cve": { + "id": "CVE-2024-1002", + "sourceIdentifier": "nvd@nist.gov", + "published": "2024-02-01T12:00:00Z", + "lastModified": "2024-02-02T12:00:00Z", + "descriptions": [ + { "lang": "en", "value": "Multipage vulnerability three." } + ], + "metrics": { + "cvssMetricV31": [ + { + "cvssData": { + "vectorString": "CVSS:3.1/AV:L/AC:H/PR:N/UI:N/S:U/C:L/I:N/A:N", + "baseScore": 3.1, + "baseSeverity": "LOW" + } + } + ] + }, + "configurations": { + "nodes": [ + { + "cpeMatch": [ + { "vulnerable": true, "criteria": "cpe:2.3:a:example:product_c:3.0:*:*:*:*:*:*:*" } + ] + } + ] + } + } + }, + { + "cve": { + "id": "CVE-2024-1003", + "sourceIdentifier": "nvd@nist.gov", + "published": "2024-02-01T13:00:00Z", + "lastModified": "2024-02-02T13:00:00Z", + "descriptions": [ + { "lang": "en", "value": "Multipage vulnerability four." } + ], + "metrics": { + "cvssMetricV31": [ + { + "cvssData": { + "vectorString": "CVSS:3.1/AV:A/AC:L/PR:N/UI:N/S:U/C:M/I:L/A:L", + "baseScore": 7.4, + "baseSeverity": "HIGH" + } + } + ] + }, + "configurations": { + "nodes": [ + { + "cpeMatch": [ + { "vulnerable": true, "criteria": "cpe:2.3:a:example:product_d:4.0:*:*:*:*:*:*:*" } + ] + } + ] + } + } + } + ] +} diff --git a/src/StellaOps.Feedser.Source.Nvd.Tests/Nvd/Fixtures/nvd-multipage-3.json b/src/StellaOps.Feedser.Source.Nvd.Tests/Nvd/Fixtures/nvd-multipage-3.json new file mode 100644 index 00000000..9b0df922 --- /dev/null +++ b/src/StellaOps.Feedser.Source.Nvd.Tests/Nvd/Fixtures/nvd-multipage-3.json @@ -0,0 +1,38 @@ +{ + "resultsPerPage": 2, + "startIndex": 4, + "totalResults": 5, + "vulnerabilities": [ + { + "cve": { + "id": "CVE-2024-1004", + "sourceIdentifier": "nvd@nist.gov", + "published": "2024-02-01T14:00:00Z", + "lastModified": "2024-02-02T14:00:00Z", + "descriptions": [ + { "lang": "en", "value": "Multipage vulnerability five." } + ], + "metrics": { + "cvssMetricV31": [ + { + "cvssData": { + "vectorString": "CVSS:3.1/AV:N/AC:H/PR:L/UI:R/S:C/C:L/I:H/A:L", + "baseScore": 7.9, + "baseSeverity": "HIGH" + } + } + ] + }, + "configurations": { + "nodes": [ + { + "cpeMatch": [ + { "vulnerable": true, "criteria": "cpe:2.3:a:example:product_e:5.0:*:*:*:*:*:*:*" } + ] + } + ] + } + } + } + ] +} diff --git a/src/StellaOps.Feedser.Source.Nvd.Tests/Nvd/Fixtures/nvd-window-1.json b/src/StellaOps.Feedser.Source.Nvd.Tests/Nvd/Fixtures/nvd-window-1.json new file mode 100644 index 00000000..1d1dd348 --- /dev/null +++ b/src/StellaOps.Feedser.Source.Nvd.Tests/Nvd/Fixtures/nvd-window-1.json @@ -0,0 +1,101 @@ +{ + "resultsPerPage": 2000, + "startIndex": 0, + "totalResults": 2, + "vulnerabilities": [ + { + "cve": { + "id": "CVE-2024-0001", + "sourceIdentifier": "nvd@nist.gov", + "published": "2024-01-01T10:00:00Z", + "lastModified": "2024-01-02T10:00:00Z", + "descriptions": [ + { "lang": "en", "value": "Example vulnerability one." } + ], + "references": [ + { + "url": "https://vendor.example.com/advisories/0001", + "source": "Vendor", + "tags": ["Vendor Advisory"] + } + ], + "weaknesses": [ + { + "description": [ + { "lang": "en", "value": "CWE-79" }, + { "lang": "en", "value": "Improper Neutralization of Input" } + ] + } + ], + "metrics": { + "cvssMetricV31": [ + { + "cvssData": { + "vectorString": "CVSS:3.1/AV:N/AC:L/PR:N/UI:N/S:U/C:H/I:H/A:H", + "baseScore": 9.8, + "baseSeverity": "CRITICAL" + } + } + ] + }, + "configurations": { + "nodes": [ + { + "cpeMatch": [ + { "vulnerable": true, "criteria": "cpe:2.3:a:example:product_one:1.0:*:*:*:*:*:*:*" } + ] + } + ] + } + } + }, + { + "cve": { + "id": "CVE-2024-0002", + "sourceIdentifier": "nvd@nist.gov", + "published": "2024-01-01T11:00:00Z", + "lastModified": "2024-01-02T11:00:00Z", + "descriptions": [ + { "lang": "fr", "value": "Description française" }, + { "lang": "en", "value": "Example vulnerability two." } + ], + "references": [ + { + "url": "https://cisa.example.gov/alerts/0002", + "source": "CISA", + "tags": ["US Government Resource"] + } + ], + "weaknesses": [ + { + "description": [ + { "lang": "en", "value": "CWE-89" }, + { "lang": "en", "value": "SQL Injection" } + ] + } + ], + "metrics": { + "cvssMetricV30": [ + { + "cvssData": { + "vectorString": "CVSS:3.0/AV:L/AC:H/PR:L/UI:R/S:U/C:L/I:L/A:L", + "baseScore": 4.6, + "baseSeverity": "MEDIUM" + } + } + ] + }, + "configurations": { + "nodes": [ + { + "cpeMatch": [ + { "vulnerable": true, "criteria": "cpe:2.3:a:example:product_two:2.0:*:*:*:*:*:*:*" }, + { "vulnerable": false, "criteria": "cpe:2.3:a:example:product_two:2.1:*:*:*:*:*:*:*" } + ] + } + ] + } + } + } + ] +} diff --git a/src/StellaOps.Feedser.Source.Nvd.Tests/Nvd/Fixtures/nvd-window-2.json b/src/StellaOps.Feedser.Source.Nvd.Tests/Nvd/Fixtures/nvd-window-2.json new file mode 100644 index 00000000..6220fe5a --- /dev/null +++ b/src/StellaOps.Feedser.Source.Nvd.Tests/Nvd/Fixtures/nvd-window-2.json @@ -0,0 +1,45 @@ +{ + "resultsPerPage": 2000, + "startIndex": 0, + "totalResults": 1, + "vulnerabilities": [ + { + "cve": { + "id": "CVE-2024-0003", + "sourceIdentifier": "nvd@nist.gov", + "published": "2024-01-01T12:00:00Z", + "lastModified": "2024-01-02T12:00:00Z", + "descriptions": [ + { "lang": "en", "value": "Example vulnerability three." } + ], + "references": [ + { + "url": "https://example.org/patches/0003", + "source": "Vendor", + "tags": ["Patch"] + } + ], + "metrics": { + "cvssMetricV2": [ + { + "cvssData": { + "vectorString": "AV:N/AC:M/Au:N/C:P/I:P/A:P", + "baseScore": 6.8, + "baseSeverity": "MEDIUM" + } + } + ] + }, + "configurations": { + "nodes": [ + { + "cpeMatch": [ + { "vulnerable": true, "criteria": "cpe:2.3:a:example:product_three:3.5:*:*:*:*:*:*:*" } + ] + } + ] + } + } + } + ] +} diff --git a/src/StellaOps.Feedser.Source.Nvd.Tests/Nvd/Fixtures/nvd-window-update.json b/src/StellaOps.Feedser.Source.Nvd.Tests/Nvd/Fixtures/nvd-window-update.json new file mode 100644 index 00000000..65ad963f --- /dev/null +++ b/src/StellaOps.Feedser.Source.Nvd.Tests/Nvd/Fixtures/nvd-window-update.json @@ -0,0 +1,51 @@ +{ + "resultsPerPage": 2000, + "startIndex": 0, + "totalResults": 1, + "vulnerabilities": [ + { + "cve": { + "id": "CVE-2024-0001", + "sourceIdentifier": "nvd@nist.gov", + "published": "2024-01-01T10:00:00Z", + "lastModified": "2024-01-03T12:00:00Z", + "descriptions": [ + { "lang": "en", "value": "Example vulnerability one updated." } + ], + "references": [ + { + "url": "https://vendor.example.com/advisories/0001", + "source": "Vendor", + "tags": ["Vendor Advisory"] + }, + { + "url": "https://kb.example.com/articles/0001", + "source": "KnowledgeBase", + "tags": ["Third Party Advisory"] + } + ], + "metrics": { + "cvssMetricV31": [ + { + "cvssData": { + "vectorString": "CVSS:3.1/AV:N/AC:L/PR:N/UI:R/S:U/C:H/I:H/A:H", + "baseScore": 8.8, + "baseSeverity": "HIGH" + } + } + ] + }, + "configurations": { + "nodes": [ + { + "cpeMatch": [ + { "vulnerable": true, "criteria": "cpe:2.3:a:example:product_one:1.0:*:*:*:*:*:*:*" }, + { "vulnerable": true, "criteria": "cpe:2.3:a:example:product_one:1.1:*:*:*:*:*:*:*" } + ] + } + ] + } + } + } + ] +} diff --git a/src/StellaOps.Feedser.Source.Nvd.Tests/Nvd/NvdConnectorHarnessTests.cs b/src/StellaOps.Feedser.Source.Nvd.Tests/Nvd/NvdConnectorHarnessTests.cs new file mode 100644 index 00000000..66ded4b4 --- /dev/null +++ b/src/StellaOps.Feedser.Source.Nvd.Tests/Nvd/NvdConnectorHarnessTests.cs @@ -0,0 +1,136 @@ +using System; +using System.Collections.Generic; +using System.Globalization; +using System.IO; +using System.Linq; +using Microsoft.Extensions.DependencyInjection; +using MongoDB.Bson; +using StellaOps.Feedser.Source.Common.Testing; +using StellaOps.Feedser.Source.Nvd; +using StellaOps.Feedser.Source.Nvd.Configuration; +using StellaOps.Feedser.Storage.Mongo; +using StellaOps.Feedser.Storage.Mongo.Advisories; +using StellaOps.Feedser.Storage.Mongo.Documents; +using StellaOps.Feedser.Testing; +using StellaOps.Feedser.Testing; +using System.Net; + +namespace StellaOps.Feedser.Source.Nvd.Tests; + +[Collection("mongo-fixture")] +public sealed class NvdConnectorHarnessTests : IAsyncLifetime +{ + private readonly ConnectorTestHarness _harness; + + public NvdConnectorHarnessTests(MongoIntegrationFixture fixture) + { + _harness = new ConnectorTestHarness(fixture, new DateTimeOffset(2024, 1, 2, 12, 0, 0, TimeSpan.Zero), NvdOptions.HttpClientName); + } + + [Fact] + public async Task FetchAsync_MultiPagePersistsStartIndexMetadata() + { + await _harness.ResetAsync(); + + var options = new NvdOptions + { + BaseEndpoint = new Uri("https://nvd.example.test/api"), + WindowSize = TimeSpan.FromHours(1), + WindowOverlap = TimeSpan.FromMinutes(5), + InitialBackfill = TimeSpan.FromHours(2), + }; + + var timeProvider = _harness.TimeProvider; + var handler = _harness.Handler; + + var windowStart = timeProvider.GetUtcNow() - options.InitialBackfill; + var windowEnd = windowStart + options.WindowSize; + + var firstUri = BuildRequestUri(options, windowStart, windowEnd); + var secondUri = BuildRequestUri(options, windowStart, windowEnd, startIndex: 2); + var thirdUri = BuildRequestUri(options, windowStart, windowEnd, startIndex: 4); + + handler.AddJsonResponse(firstUri, ReadFixture("nvd-multipage-1.json")); + handler.AddJsonResponse(secondUri, ReadFixture("nvd-multipage-2.json")); + handler.AddJsonResponse(thirdUri, ReadFixture("nvd-multipage-3.json")); + + await _harness.EnsureServiceProviderAsync(services => + { + services.AddNvdConnector(opts => + { + opts.BaseEndpoint = options.BaseEndpoint; + opts.WindowSize = options.WindowSize; + opts.WindowOverlap = options.WindowOverlap; + opts.InitialBackfill = options.InitialBackfill; + }); + }); + + var provider = _harness.ServiceProvider; + var connector = new NvdConnectorPlugin().Create(provider); + + await connector.FetchAsync(provider, CancellationToken.None); + + var documentStore = provider.GetRequiredService(); + + var firstDocument = await documentStore.FindBySourceAndUriAsync(NvdConnectorPlugin.SourceName, firstUri.ToString(), CancellationToken.None); + Assert.NotNull(firstDocument); + Assert.Equal("0", firstDocument!.Metadata["startIndex"]); + + var secondDocument = await documentStore.FindBySourceAndUriAsync(NvdConnectorPlugin.SourceName, secondUri.ToString(), CancellationToken.None); + Assert.NotNull(secondDocument); + Assert.Equal("2", secondDocument!.Metadata["startIndex"]); + + var thirdDocument = await documentStore.FindBySourceAndUriAsync(NvdConnectorPlugin.SourceName, thirdUri.ToString(), CancellationToken.None); + Assert.NotNull(thirdDocument); + Assert.Equal("4", thirdDocument!.Metadata["startIndex"]); + + var stateRepository = provider.GetRequiredService(); + var state = await stateRepository.TryGetAsync(NvdConnectorPlugin.SourceName, CancellationToken.None); + Assert.NotNull(state); + var pendingDocuments = state!.Cursor.TryGetValue("pendingDocuments", out var pending) + ? pending.AsBsonArray + : new BsonArray(); + Assert.Equal(3, pendingDocuments.Count); + } + + public Task InitializeAsync() => Task.CompletedTask; + + public Task DisposeAsync() => _harness.ResetAsync(); + + private static Uri BuildRequestUri(NvdOptions options, DateTimeOffset start, DateTimeOffset end, int startIndex = 0) + { + var builder = new UriBuilder(options.BaseEndpoint); + var parameters = new Dictionary + { + ["lastModifiedStartDate"] = start.ToString("yyyy-MM-dd'T'HH:mm:ss.fffK"), + ["lastModifiedEndDate"] = end.ToString("yyyy-MM-dd'T'HH:mm:ss.fffK"), + ["resultsPerPage"] = "2000", + }; + + if (startIndex > 0) + { + parameters["startIndex"] = startIndex.ToString(CultureInfo.InvariantCulture); + } + + builder.Query = string.Join("&", parameters.Select(kvp => $"{WebUtility.UrlEncode(kvp.Key)}={WebUtility.UrlEncode(kvp.Value)}")); + return builder.Uri; + } + + private static string ReadFixture(string filename) + { + var baseDirectory = AppContext.BaseDirectory; + var primary = Path.Combine(baseDirectory, "Source", "Nvd", "Fixtures", filename); + if (File.Exists(primary)) + { + return File.ReadAllText(primary); + } + + var secondary = Path.Combine(baseDirectory, "Nvd", "Fixtures", filename); + if (File.Exists(secondary)) + { + return File.ReadAllText(secondary); + } + + throw new FileNotFoundException($"Fixture '{filename}' was not found in the test output directory."); + } +} diff --git a/src/StellaOps.Feedser.Source.Nvd.Tests/Nvd/NvdConnectorTests.cs b/src/StellaOps.Feedser.Source.Nvd.Tests/Nvd/NvdConnectorTests.cs new file mode 100644 index 00000000..14554ef3 --- /dev/null +++ b/src/StellaOps.Feedser.Source.Nvd.Tests/Nvd/NvdConnectorTests.cs @@ -0,0 +1,653 @@ +using System; +using System.Collections.Concurrent; +using System.Globalization; +using System.IO; +using System.Linq; +using System.Net; +using System.Diagnostics.Metrics; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Http; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Logging.Abstractions; +using Microsoft.Extensions.Options; +using Microsoft.Extensions.Time.Testing; +using MongoDB.Bson; +using StellaOps.Feedser.Models; +using StellaOps.Feedser.Source.Common.Fetch; +using StellaOps.Feedser.Source.Common.Http; +using StellaOps.Feedser.Source.Common.Testing; +using StellaOps.Feedser.Source.Common; +using StellaOps.Feedser.Source.Nvd; +using StellaOps.Feedser.Source.Nvd.Configuration; +using StellaOps.Feedser.Source.Nvd.Internal; +using StellaOps.Feedser.Storage.Mongo; +using StellaOps.Feedser.Storage.Mongo.Advisories; +using StellaOps.Feedser.Storage.Mongo.Documents; +using StellaOps.Feedser.Storage.Mongo.Dtos; +using StellaOps.Feedser.Storage.Mongo.ChangeHistory; +using StellaOps.Feedser.Testing; + +namespace StellaOps.Feedser.Source.Nvd.Tests; + +[Collection("mongo-fixture")] +public sealed class NvdConnectorTests : IAsyncLifetime +{ + private readonly MongoIntegrationFixture _fixture; + private FakeTimeProvider _timeProvider; + private readonly DateTimeOffset _initialNow; + private readonly CannedHttpMessageHandler _handler; + private ServiceProvider? _serviceProvider; + + public NvdConnectorTests(MongoIntegrationFixture fixture) + { + _fixture = fixture; + _initialNow = new DateTimeOffset(2024, 1, 2, 12, 0, 0, TimeSpan.Zero); + _timeProvider = new FakeTimeProvider(_initialNow); + _handler = new CannedHttpMessageHandler(); + } + + [Fact] + public async Task FetchParseMap_FlowProducesCanonicalAdvisories() + { + await ResetDatabaseAsync(); + + var options = new NvdOptions + { + BaseEndpoint = new Uri("https://nvd.example.test/api"), + WindowSize = TimeSpan.FromHours(1), + WindowOverlap = TimeSpan.FromMinutes(5), + InitialBackfill = TimeSpan.FromHours(2), + }; + + var window1Start = _timeProvider.GetUtcNow() - options.InitialBackfill; + var window1End = window1Start + options.WindowSize; + _handler.AddJsonResponse(BuildRequestUri(options, window1Start, window1End), ReadFixture("nvd-window-1.json")); + + await EnsureServiceProviderAsync(options); + var provider = _serviceProvider!; + + var connector = new NvdConnectorPlugin().Create(provider); + + await connector.FetchAsync(provider, CancellationToken.None); + await connector.ParseAsync(provider, CancellationToken.None); + await connector.MapAsync(provider, CancellationToken.None); + + var advisoryStore = provider.GetRequiredService(); + var advisories = await advisoryStore.GetRecentAsync(10, CancellationToken.None); + Assert.Contains(advisories, advisory => advisory.AdvisoryKey == "CVE-2024-0001"); + Assert.Contains(advisories, advisory => advisory.AdvisoryKey == "CVE-2024-0002"); + + var cve1 = advisories.Single(advisory => advisory.AdvisoryKey == "CVE-2024-0001"); + var package1 = Assert.Single(cve1.AffectedPackages); + var range1 = Assert.Single(package1.VersionRanges); + Assert.Equal("cpe", range1.RangeKind); + Assert.Equal("1.0", range1.IntroducedVersion); + Assert.Null(range1.FixedVersion); + Assert.Equal("1.0", range1.LastAffectedVersion); + Assert.Equal("==1.0", range1.RangeExpression); + Assert.NotNull(range1.Primitives); + Assert.Equal("1.0", range1.Primitives!.VendorExtensions!["version"]); + Assert.Contains(cve1.References, reference => reference.Kind == "weakness" && reference.SourceTag == "CWE-79"); + var cvss1 = Assert.Single(cve1.CvssMetrics); + Assert.Equal("CVSS:3.1/AV:N/AC:L/PR:N/UI:N/S:U/C:H/I:H/A:H", cvss1.Provenance.Value); + + var cve2 = advisories.Single(advisory => advisory.AdvisoryKey == "CVE-2024-0002"); + var package2 = Assert.Single(cve2.AffectedPackages); + var range2 = Assert.Single(package2.VersionRanges); + Assert.Equal("cpe", range2.RangeKind); + Assert.Equal("2.0", range2.IntroducedVersion); + Assert.Null(range2.FixedVersion); + Assert.Equal("2.0", range2.LastAffectedVersion); + Assert.Equal("==2.0", range2.RangeExpression); + Assert.NotNull(range2.Primitives); + Assert.Equal("2.0", range2.Primitives!.VendorExtensions!["version"]); + Assert.Contains(cve2.References, reference => reference.Kind == "weakness" && reference.SourceTag == "CWE-89"); + var cvss2 = Assert.Single(cve2.CvssMetrics); + Assert.Equal("CVSS:3.0/AV:L/AC:H/PR:L/UI:R/S:U/C:L/I:L/A:L", cvss2.Provenance.Value); + + var stateRepository = provider.GetRequiredService(); + var state = await stateRepository.TryGetAsync(NvdConnectorPlugin.SourceName, CancellationToken.None); + Assert.NotNull(state); + var cursorDocument = state!.Cursor; + Assert.NotNull(cursorDocument); + var lastWindowEnd = cursorDocument.TryGetValue("windowEnd", out var endValue) ? ReadDateTime(endValue) : (DateTimeOffset?)null; + Assert.Equal(window1End.UtcDateTime, lastWindowEnd?.UtcDateTime); + + _timeProvider.Advance(TimeSpan.FromHours(1)); + var now = _timeProvider.GetUtcNow(); + var startCandidate = (lastWindowEnd ?? window1End) - options.WindowOverlap; + var backfillLimit = now - options.InitialBackfill; + var window2Start = startCandidate < backfillLimit ? backfillLimit : startCandidate; + var window2End = window2Start + options.WindowSize; + if (window2End > now) + { + window2End = now; + } + + _handler.AddJsonResponse(BuildRequestUri(options, window2Start, window2End), ReadFixture("nvd-window-2.json")); + + await connector.FetchAsync(provider, CancellationToken.None); + await connector.ParseAsync(provider, CancellationToken.None); + await connector.MapAsync(provider, CancellationToken.None); + + advisories = await advisoryStore.GetRecentAsync(10, CancellationToken.None); + Assert.Equal(3, advisories.Count); + Assert.Contains(advisories, advisory => advisory.AdvisoryKey == "CVE-2024-0003"); + var cve3 = advisories.Single(advisory => advisory.AdvisoryKey == "CVE-2024-0003"); + var package3 = Assert.Single(cve3.AffectedPackages); + var range3 = Assert.Single(package3.VersionRanges); + Assert.Equal("3.5", range3.IntroducedVersion); + Assert.Equal("3.5", range3.LastAffectedVersion); + Assert.Equal("==3.5", range3.RangeExpression); + Assert.NotNull(range3.Primitives); + Assert.Equal("3.5", range3.Primitives!.VendorExtensions!["version"]); + + var documentStore = provider.GetRequiredService(); + var finalState = await stateRepository.TryGetAsync(NvdConnectorPlugin.SourceName, CancellationToken.None); + Assert.NotNull(finalState); + var pendingDocuments = finalState!.Cursor.TryGetValue("pendingDocuments", out var pendingDocs) + ? pendingDocs.AsBsonArray + : new BsonArray(); + Assert.Empty(pendingDocuments); + } + + [Fact] + public async Task FetchAsync_MultiPageWindowFetchesAllPages() + { + await ResetDatabaseAsync(); + + var options = new NvdOptions + { + BaseEndpoint = new Uri("https://nvd.example.test/api"), + WindowSize = TimeSpan.FromHours(1), + WindowOverlap = TimeSpan.FromMinutes(5), + InitialBackfill = TimeSpan.FromHours(2), + }; + + var windowStart = _timeProvider.GetUtcNow() - options.InitialBackfill; + var windowEnd = windowStart + options.WindowSize; + + _handler.AddJsonResponse(BuildRequestUri(options, windowStart, windowEnd), ReadFixture("nvd-multipage-1.json")); + _handler.AddJsonResponse(BuildRequestUri(options, windowStart, windowEnd, startIndex: 2), ReadFixture("nvd-multipage-2.json")); + _handler.AddJsonResponse(BuildRequestUri(options, windowStart, windowEnd, startIndex: 4), ReadFixture("nvd-multipage-3.json")); + _handler.AddJsonResponse(BuildRequestUri(options, windowStart, windowEnd, startIndex: 4), ReadFixture("nvd-multipage-3.json")); + _handler.AddJsonResponse(BuildRequestUri(options, windowStart, windowEnd, startIndex: 4), ReadFixture("nvd-multipage-3.json")); + _handler.AddJsonResponse(BuildRequestUri(options, windowStart, windowEnd, startIndex: 4), ReadFixture("nvd-multipage-3.json")); + _handler.AddJsonResponse(BuildRequestUri(options, windowStart, windowEnd, startIndex: 4), ReadFixture("nvd-multipage-3.json")); + _handler.AddJsonResponse(BuildRequestUri(options, windowStart, windowEnd, startIndex: 4), ReadFixture("nvd-multipage-3.json")); + _handler.AddJsonResponse(BuildRequestUri(options, windowStart, windowEnd, startIndex: 4), ReadFixture("nvd-multipage-3.json")); + _handler.AddJsonResponse(BuildRequestUri(options, windowStart, windowEnd, startIndex: 4), ReadFixture("nvd-multipage-3.json")); + + await EnsureServiceProviderAsync(options); + var provider = _serviceProvider!; + var connector = new NvdConnectorPlugin().Create(provider); + + await connector.FetchAsync(provider, CancellationToken.None); + + var stateRepository = provider.GetRequiredService(); + var state = await stateRepository.TryGetAsync(NvdConnectorPlugin.SourceName, CancellationToken.None); + Assert.NotNull(state); + var pendingDocuments = state!.Cursor.TryGetValue("pendingDocuments", out var pendingDocs) + ? pendingDocs.AsBsonArray.Select(v => Guid.Parse(v.AsString)).ToArray() + : Array.Empty(); + Assert.Equal(3, pendingDocuments.Length); + + await connector.ParseAsync(provider, CancellationToken.None); + await connector.MapAsync(provider, CancellationToken.None); + + var advisoryStore = provider.GetRequiredService(); + var advisories = await advisoryStore.GetRecentAsync(10, CancellationToken.None); + var advisoryKeys = advisories.Select(advisory => advisory.AdvisoryKey).OrderBy(k => k).ToArray(); + + Assert.Equal(new[] { "CVE-2024-1000", "CVE-2024-1001", "CVE-2024-1002", "CVE-2024-1003", "CVE-2024-1004" }, advisoryKeys); + } + + [Fact] + public async Task Observability_RecordsCountersForSuccessfulFlow() + { + await ResetDatabaseAsync(); + + var options = new NvdOptions + { + BaseEndpoint = new Uri("https://nvd.example.test/api"), + WindowSize = TimeSpan.FromHours(1), + WindowOverlap = TimeSpan.FromMinutes(5), + InitialBackfill = TimeSpan.FromHours(2), + }; + + using var collector = new MetricCollector(NvdDiagnostics.MeterName); + + var windowStart = _timeProvider.GetUtcNow() - options.InitialBackfill; + var windowEnd = windowStart + options.WindowSize; + + var handler = new CannedHttpMessageHandler(); + handler.AddJsonResponse(BuildRequestUri(options, windowStart, windowEnd), ReadFixture("nvd-multipage-1.json")); + handler.AddJsonResponse(BuildRequestUri(options, windowStart, windowEnd, startIndex: 2), ReadFixture("nvd-multipage-2.json")); + handler.AddJsonResponse(BuildRequestUri(options, windowStart, windowEnd, startIndex: 4), ReadFixture("nvd-multipage-3.json")); + handler.AddJsonResponse(BuildRequestUri(options, windowStart, windowEnd, startIndex: 4), ReadFixture("nvd-multipage-3.json")); + handler.AddJsonResponse(BuildRequestUri(options, windowStart, windowEnd, startIndex: 4), ReadFixture("nvd-multipage-3.json")); + + await using var provider = await CreateServiceProviderAsync(options, handler); + var connector = new NvdConnectorPlugin().Create(provider); + + await connector.FetchAsync(provider, CancellationToken.None); + await connector.ParseAsync(provider, CancellationToken.None); + await connector.MapAsync(provider, CancellationToken.None); + + Assert.Equal(3, collector.GetValue("nvd.fetch.attempts")); + Assert.Equal(3, collector.GetValue("nvd.fetch.documents")); + Assert.Equal(0, collector.GetValue("nvd.fetch.failures")); + Assert.Equal(0, collector.GetValue("nvd.fetch.unchanged")); + Assert.Equal(3, collector.GetValue("nvd.parse.success")); + Assert.Equal(0, collector.GetValue("nvd.parse.failures")); + Assert.Equal(0, collector.GetValue("nvd.parse.quarantine")); + Assert.Equal(5, collector.GetValue("nvd.map.success")); + } + + [Fact] + public async Task ChangeHistory_RecordsDifferencesForModifiedCve() + { + await ResetDatabaseAsync(); + + var options = new NvdOptions + { + BaseEndpoint = new Uri("https://nvd.example.test/api"), + WindowSize = TimeSpan.FromHours(1), + WindowOverlap = TimeSpan.FromMinutes(5), + InitialBackfill = TimeSpan.FromHours(2), + }; + + var windowStart = _timeProvider.GetUtcNow() - options.InitialBackfill; + var windowEnd = windowStart + options.WindowSize; + _handler.AddJsonResponse(BuildRequestUri(options, windowStart, windowEnd), ReadFixture("nvd-window-1.json")); + + await EnsureServiceProviderAsync(options); + var provider = _serviceProvider!; + var connector = new NvdConnectorPlugin().Create(provider); + + await connector.FetchAsync(provider, CancellationToken.None); + await connector.ParseAsync(provider, CancellationToken.None); + await connector.MapAsync(provider, CancellationToken.None); + + var historyStore = provider.GetRequiredService(); + var historyEntries = await historyStore.GetRecentAsync("nvd", "CVE-2024-0001", 5, CancellationToken.None); + Assert.Empty(historyEntries); + + _timeProvider.Advance(TimeSpan.FromHours(2)); + var now = _timeProvider.GetUtcNow(); + + var stateRepository = provider.GetRequiredService(); + var state = await stateRepository.TryGetAsync(NvdConnectorPlugin.SourceName, CancellationToken.None); + Assert.NotNull(state); + + var cursorDocument = state!.Cursor; + var lastWindowEnd = cursorDocument.TryGetValue("windowEnd", out var endValue) ? ReadDateTime(endValue) : (DateTimeOffset?)null; + var startCandidate = (lastWindowEnd ?? windowEnd) - options.WindowOverlap; + var backfillLimit = now - options.InitialBackfill; + var window2Start = startCandidate < backfillLimit ? backfillLimit : startCandidate; + var window2End = window2Start + options.WindowSize; + if (window2End > now) + { + window2End = now; + } + + _handler.AddJsonResponse(BuildRequestUri(options, window2Start, window2End), ReadFixture("nvd-window-update.json")); + + await connector.FetchAsync(provider, CancellationToken.None); + await connector.ParseAsync(provider, CancellationToken.None); + await connector.MapAsync(provider, CancellationToken.None); + + var advisoryStore = provider.GetRequiredService(); + var updatedAdvisory = await advisoryStore.FindAsync("CVE-2024-0001", CancellationToken.None); + Assert.NotNull(updatedAdvisory); + Assert.Equal("high", updatedAdvisory!.Severity); + + historyEntries = await historyStore.GetRecentAsync("nvd", "CVE-2024-0001", 5, CancellationToken.None); + Assert.NotEmpty(historyEntries); + var latest = historyEntries[0]; + Assert.Equal("nvd", latest.SourceName); + Assert.Equal("CVE-2024-0001", latest.AdvisoryKey); + Assert.NotNull(latest.PreviousHash); + Assert.NotEqual(latest.PreviousHash, latest.CurrentHash); + Assert.Contains(latest.Changes, change => change.Field == "severity" && change.ChangeType == "Modified"); + Assert.Contains(latest.Changes, change => change.Field == "references" && change.ChangeType == "Modified"); + } + + [Fact] + public async Task ParseAsync_InvalidSchema_QuarantinesDocumentAndEmitsMetric() + { + await ResetDatabaseAsync(); + + var options = new NvdOptions + { + BaseEndpoint = new Uri("https://nvd.example.test/api"), + WindowSize = TimeSpan.FromHours(1), + WindowOverlap = TimeSpan.FromMinutes(5), + InitialBackfill = TimeSpan.FromHours(2), + }; + + using var collector = new MetricCollector(NvdDiagnostics.MeterName); + + var windowStart = _timeProvider.GetUtcNow() - options.InitialBackfill; + var windowEnd = windowStart + options.WindowSize; + var requestUri = BuildRequestUri(options, windowStart, windowEnd); + + _handler.AddJsonResponse(requestUri, ReadFixture("nvd-invalid-schema.json")); + + await EnsureServiceProviderAsync(options); + var provider = _serviceProvider!; + var connector = new NvdConnectorPlugin().Create(provider); + + await connector.FetchAsync(provider, CancellationToken.None); + await connector.ParseAsync(provider, CancellationToken.None); + + var documentStore = provider.GetRequiredService(); + var document = await documentStore.FindBySourceAndUriAsync(NvdConnectorPlugin.SourceName, requestUri.ToString(), CancellationToken.None); + Assert.NotNull(document); + Assert.Equal(DocumentStatuses.Failed, document!.Status); + + var stateRepository = provider.GetRequiredService(); + var state = await stateRepository.TryGetAsync(NvdConnectorPlugin.SourceName, CancellationToken.None); + Assert.NotNull(state); + var pendingDocs = state!.Cursor.TryGetValue("pendingDocuments", out var pendingDocsValue) ? pendingDocsValue.AsBsonArray : new BsonArray(); + Assert.Empty(pendingDocs); + var pendingMappings = state.Cursor.TryGetValue("pendingMappings", out var pendingMappingsValue) ? pendingMappingsValue.AsBsonArray : new BsonArray(); + Assert.Empty(pendingMappings); + + Assert.Equal(1, collector.GetValue("nvd.fetch.documents")); + Assert.Equal(0, collector.GetValue("nvd.parse.success")); + Assert.Equal(1, collector.GetValue("nvd.parse.quarantine")); + Assert.Equal(0, collector.GetValue("nvd.map.success")); + } + + [Fact] + public async Task ResetDatabase_IsolatesRuns() + { + await ResetDatabaseAsync(); + + var options = new NvdOptions + { + BaseEndpoint = new Uri("https://nvd.example.test/api"), + WindowSize = TimeSpan.FromHours(1), + WindowOverlap = TimeSpan.FromMinutes(5), + InitialBackfill = TimeSpan.FromHours(2), + }; + + var start = _timeProvider.GetUtcNow() - options.InitialBackfill; + var end = start + options.WindowSize; + _handler.AddJsonResponse(BuildRequestUri(options, start, end), ReadFixture("nvd-window-1.json")); + + await EnsureServiceProviderAsync(options); + var provider = _serviceProvider!; + var connector = new NvdConnectorPlugin().Create(provider); + + await connector.FetchAsync(provider, CancellationToken.None); + await connector.ParseAsync(provider, CancellationToken.None); + await connector.MapAsync(provider, CancellationToken.None); + + var advisoryStore = provider.GetRequiredService(); + var firstRunKeys = (await advisoryStore.GetRecentAsync(10, CancellationToken.None)) + .Select(advisory => advisory.AdvisoryKey) + .OrderBy(k => k) + .ToArray(); + Assert.Equal(new[] { "CVE-2024-0001", "CVE-2024-0002" }, firstRunKeys); + + await ResetDatabaseAsync(); + + options = new NvdOptions + { + BaseEndpoint = new Uri("https://nvd.example.test/api"), + WindowSize = TimeSpan.FromHours(1), + WindowOverlap = TimeSpan.FromMinutes(5), + InitialBackfill = TimeSpan.FromHours(2), + }; + + start = _timeProvider.GetUtcNow() - options.InitialBackfill; + end = start + options.WindowSize; + _handler.AddJsonResponse(BuildRequestUri(options, start, end), ReadFixture("nvd-window-2.json")); + + await EnsureServiceProviderAsync(options); + provider = _serviceProvider!; + connector = new NvdConnectorPlugin().Create(provider); + + await connector.FetchAsync(provider, CancellationToken.None); + await connector.ParseAsync(provider, CancellationToken.None); + await connector.MapAsync(provider, CancellationToken.None); + + advisoryStore = provider.GetRequiredService(); + var secondRunKeys = (await advisoryStore.GetRecentAsync(10, CancellationToken.None)) + .Select(advisory => advisory.AdvisoryKey) + .OrderBy(k => k) + .ToArray(); + Assert.Equal(new[] { "CVE-2024-0003" }, secondRunKeys); + } + + private async Task EnsureServiceProviderAsync(NvdOptions options) + { + if (_serviceProvider is not null) + { + return; + } + + _serviceProvider = await CreateServiceProviderAsync(options, _handler); + } + + [Fact] + public async Task Resume_CompletesPendingDocumentsAfterRestart() + { + await ResetDatabaseAsync(); + + var options = new NvdOptions + { + BaseEndpoint = new Uri("https://nvd.example.test/api"), + WindowSize = TimeSpan.FromHours(1), + WindowOverlap = TimeSpan.FromMinutes(5), + InitialBackfill = TimeSpan.FromHours(2), + }; + + var windowStart = _timeProvider.GetUtcNow() - options.InitialBackfill; + var windowEnd = windowStart + options.WindowSize; + var requestUri = BuildRequestUri(options, windowStart, windowEnd); + + var fetchHandler = new CannedHttpMessageHandler(); + fetchHandler.AddJsonResponse(requestUri, ReadFixture("nvd-window-1.json")); + + Guid[] pendingDocumentIds; + await using (var fetchProvider = await CreateServiceProviderAsync(options, fetchHandler)) + { + var connector = new NvdConnectorPlugin().Create(fetchProvider); + await connector.FetchAsync(fetchProvider, CancellationToken.None); + + var stateRepository = fetchProvider.GetRequiredService(); + var state = await stateRepository.TryGetAsync(NvdConnectorPlugin.SourceName, CancellationToken.None); + Assert.NotNull(state); + var pending = state!.Cursor.TryGetValue("pendingDocuments", out var value) + ? value.AsBsonArray + : new BsonArray(); + Assert.NotEmpty(pending); + pendingDocumentIds = pending.Select(v => Guid.Parse(v.AsString)).ToArray(); + } + + var resumeHandler = new CannedHttpMessageHandler(); + await using (var resumeProvider = await CreateServiceProviderAsync(options, resumeHandler)) + { + var resumeConnector = new NvdConnectorPlugin().Create(resumeProvider); + + await resumeConnector.ParseAsync(resumeProvider, CancellationToken.None); + await resumeConnector.MapAsync(resumeProvider, CancellationToken.None); + + var documentStore = resumeProvider.GetRequiredService(); + foreach (var documentId in pendingDocumentIds) + { + var document = await documentStore.FindAsync(documentId, CancellationToken.None); + Assert.NotNull(document); + Assert.Equal(DocumentStatuses.Mapped, document!.Status); + } + + var advisoryStore = resumeProvider.GetRequiredService(); + var advisories = await advisoryStore.GetRecentAsync(10, CancellationToken.None); + Assert.NotEmpty(advisories); + + var stateRepository = resumeProvider.GetRequiredService(); + var finalState = await stateRepository.TryGetAsync(NvdConnectorPlugin.SourceName, CancellationToken.None); + Assert.NotNull(finalState); + var cursor = finalState!.Cursor; + var finalPendingDocs = cursor.TryGetValue("pendingDocuments", out var pendingDocs) ? pendingDocs.AsBsonArray : new BsonArray(); + Assert.Empty(finalPendingDocs); + var finalPendingMappings = cursor.TryGetValue("pendingMappings", out var pendingMappings) ? pendingMappings.AsBsonArray : new BsonArray(); + Assert.Empty(finalPendingMappings); + } + } + + private Task ResetDatabaseAsync() + { + return ResetDatabaseInternalAsync(); + } + + private async Task CreateServiceProviderAsync(NvdOptions options, CannedHttpMessageHandler handler) + { + var services = new ServiceCollection(); + services.AddLogging(builder => builder.AddProvider(NullLoggerProvider.Instance)); + services.AddSingleton(_timeProvider); + services.AddSingleton(handler); + + services.AddMongoStorage(storageOptions => + { + storageOptions.ConnectionString = _fixture.Runner.ConnectionString; + storageOptions.DatabaseName = _fixture.Database.DatabaseNamespace.DatabaseName; + storageOptions.CommandTimeout = TimeSpan.FromSeconds(5); + }); + + services.AddSourceCommon(); + services.AddNvdConnector(configure: opts => + { + opts.BaseEndpoint = options.BaseEndpoint; + opts.WindowSize = options.WindowSize; + opts.WindowOverlap = options.WindowOverlap; + opts.InitialBackfill = options.InitialBackfill; + }); + + services.Configure(NvdOptions.HttpClientName, builderOptions => + { + builderOptions.HttpMessageHandlerBuilderActions.Add(builder => + { + builder.PrimaryHandler = handler; + }); + }); + + var provider = services.BuildServiceProvider(); + var bootstrapper = provider.GetRequiredService(); + await bootstrapper.InitializeAsync(CancellationToken.None); + return provider; + } + + private async Task ResetDatabaseInternalAsync() + { + if (_serviceProvider is not null) + { + if (_serviceProvider is IAsyncDisposable asyncDisposable) + { + await asyncDisposable.DisposeAsync(); + } + else + { + _serviceProvider.Dispose(); + } + + _serviceProvider = null; + } + + await _fixture.Client.DropDatabaseAsync(_fixture.Database.DatabaseNamespace.DatabaseName); + _handler.Clear(); + _timeProvider = new FakeTimeProvider(_initialNow); + } + + private sealed class MetricCollector : IDisposable + { + private readonly MeterListener _listener; + private readonly ConcurrentDictionary _measurements = new(StringComparer.OrdinalIgnoreCase); + + public MetricCollector(string meterName) + { + _listener = new MeterListener + { + InstrumentPublished = (instrument, listener) => + { + if (instrument.Meter.Name == meterName) + { + listener.EnableMeasurementEvents(instrument); + } + } + }; + + _listener.SetMeasurementEventCallback((instrument, measurement, tags, state) => + { + _measurements.AddOrUpdate(instrument.Name, measurement, (_, existing) => existing + measurement); + }); + + _listener.Start(); + } + + public long GetValue(string instrumentName) + => _measurements.TryGetValue(instrumentName, out var value) ? value : 0; + + public void Dispose() + { + _listener.Dispose(); + } + } + + private static Uri BuildRequestUri(NvdOptions options, DateTimeOffset start, DateTimeOffset end, int startIndex = 0) + { + var builder = new UriBuilder(options.BaseEndpoint); + var parameters = new Dictionary + { + ["lastModifiedStartDate"] = start.ToString("yyyy-MM-dd'T'HH:mm:ss.fffK"), + ["lastModifiedEndDate"] = end.ToString("yyyy-MM-dd'T'HH:mm:ss.fffK"), + ["resultsPerPage"] = "2000", + }; + + if (startIndex > 0) + { + parameters["startIndex"] = startIndex.ToString(CultureInfo.InvariantCulture); + } + + builder.Query = string.Join("&", parameters.Select(static kvp => $"{System.Net.WebUtility.UrlEncode(kvp.Key)}={System.Net.WebUtility.UrlEncode(kvp.Value)}")); + return builder.Uri; + } + + private static DateTimeOffset? ReadDateTime(BsonValue value) + { + return value.BsonType switch + { + BsonType.DateTime => DateTime.SpecifyKind(value.ToUniversalTime(), DateTimeKind.Utc), + BsonType.String when DateTimeOffset.TryParse(value.AsString, out var parsed) => parsed.ToUniversalTime(), + _ => null, + }; + } + + private static string ReadFixture(string filename) + { + var baseDirectory = AppContext.BaseDirectory; + var primary = Path.Combine(baseDirectory, "Source", "Nvd", "Fixtures", filename); + if (File.Exists(primary)) + { + return File.ReadAllText(primary); + } + + var secondary = Path.Combine(baseDirectory, "Nvd", "Fixtures", filename); + if (File.Exists(secondary)) + { + return File.ReadAllText(secondary); + } + + throw new FileNotFoundException($"Fixture '{filename}' was not found in the test output directory."); + } + + public Task InitializeAsync() => Task.CompletedTask; + + public async Task DisposeAsync() + { + await ResetDatabaseInternalAsync(); + } +} diff --git a/src/StellaOps.Feedser.Source.Nvd.Tests/StellaOps.Feedser.Source.Nvd.Tests.csproj b/src/StellaOps.Feedser.Source.Nvd.Tests/StellaOps.Feedser.Source.Nvd.Tests.csproj new file mode 100644 index 00000000..3b8c3045 --- /dev/null +++ b/src/StellaOps.Feedser.Source.Nvd.Tests/StellaOps.Feedser.Source.Nvd.Tests.csproj @@ -0,0 +1,16 @@ + + + net10.0 + enable + enable + + + + + + + + + + + diff --git a/src/StellaOps.Feedser.Source.Nvd/AGENTS.md b/src/StellaOps.Feedser.Source.Nvd/AGENTS.md new file mode 100644 index 00000000..a22b7a18 --- /dev/null +++ b/src/StellaOps.Feedser.Source.Nvd/AGENTS.md @@ -0,0 +1,26 @@ +# AGENTS +## Role +Connector for NVD API v2: fetch, validate, map CVE items to canonical advisories, including CVSS/CWE/CPE as aliases/references. +## Scope +- Windowed fetch by modified range (6-12h default) with pagination; respect rate limits. +- Parse NVD JSON; validate against schema; extract CVSS v3/v4 metrics, CWE IDs, configurations.cpeMatch. +- Map to Advisory: primary id='CVE-YYYY-NNNN'; references; AffectedPackage entries for CPE (type=cpe) and optional vendor tags. +- Optional change-history capture: store previous payload hashes and diff summaries for auditing modified CVEs. +- Watermark: last successful modified_end; handle partial windows with overlap to avoid misses. +## Participants +- Merge engine reconciles NVD with PSIRT/OVAL (NVD yields to OVAL for OS packages). +- KEV connector may flag some CVEs; NVD severity is preserved but not overridden by KEV. +- Exporters consume canonical advisories. +## Interfaces & contracts +- Job kinds: nvd:fetch, nvd:parse, nvd:map. +- Input params: windowHours, since, until; safe defaults in FeedserOptions. +- Output: raw documents, sanitized DTOs, mapped advisories + provenance (document, parser). +## In/Out of scope +In: registry-level data, references, generic CPEs. +Out: authoritative distro package ranges; vendor patch states. +## Observability & security expectations +- Metrics: SourceDiagnostics publishes `feedser.source.http.*` counters/histograms tagged `feedser.source=nvd`; dashboards slice on the tag to track page counts, schema failures, map throughput, and window advancement. Structured logs include window bounds and etag hits. +## Tests +- Author and review coverage in `../StellaOps.Feedser.Source.Nvd.Tests`. +- Shared fixtures (e.g., `MongoIntegrationFixture`, `ConnectorTestHarness`) live in `../StellaOps.Feedser.Testing`. +- Keep fixtures deterministic; match new cases to real-world advisories or regression scenarios. diff --git a/src/StellaOps.Feedser.Source.Nvd/Configuration/NvdOptions.cs b/src/StellaOps.Feedser.Source.Nvd/Configuration/NvdOptions.cs new file mode 100644 index 00000000..6d26c6df --- /dev/null +++ b/src/StellaOps.Feedser.Source.Nvd/Configuration/NvdOptions.cs @@ -0,0 +1,57 @@ +namespace StellaOps.Feedser.Source.Nvd.Configuration; + +public sealed class NvdOptions +{ + /// + /// Name of the HttpClient registered for NVD fetches. + /// + public const string HttpClientName = "nvd"; + + /// + /// Base API endpoint for CVE feed queries. + /// + public Uri BaseEndpoint { get; set; } = new("https://services.nvd.nist.gov/rest/json/cves/2.0"); + + /// + /// Duration of each modified window fetch. + /// + public TimeSpan WindowSize { get; set; } = TimeSpan.FromHours(4); + + /// + /// Overlap added when advancing the sliding window to cover upstream delays. + /// + public TimeSpan WindowOverlap { get; set; } = TimeSpan.FromMinutes(5); + + /// + /// Maximum look-back period used when the connector first starts or state is empty. + /// + public TimeSpan InitialBackfill { get; set; } = TimeSpan.FromDays(7); + + public void Validate() + { + if (BaseEndpoint is null) + { + throw new InvalidOperationException("NVD base endpoint must be configured."); + } + + if (!BaseEndpoint.IsAbsoluteUri) + { + throw new InvalidOperationException("NVD base endpoint must be an absolute URI."); + } + + if (WindowSize <= TimeSpan.Zero) + { + throw new InvalidOperationException("Window size must be positive."); + } + + if (WindowOverlap < TimeSpan.Zero || WindowOverlap >= WindowSize) + { + throw new InvalidOperationException("Window overlap must be non-negative and less than the window size."); + } + + if (InitialBackfill <= TimeSpan.Zero) + { + throw new InvalidOperationException("Initial backfill duration must be positive."); + } + } +} diff --git a/src/StellaOps.Feedser.Source.Nvd/Internal/NvdCursor.cs b/src/StellaOps.Feedser.Source.Nvd/Internal/NvdCursor.cs new file mode 100644 index 00000000..01cb5e6c --- /dev/null +++ b/src/StellaOps.Feedser.Source.Nvd/Internal/NvdCursor.cs @@ -0,0 +1,64 @@ +using System.Linq; +using MongoDB.Bson; +using StellaOps.Feedser.Source.Common.Cursors; + +namespace StellaOps.Feedser.Source.Nvd.Internal; + +internal sealed record NvdCursor( + TimeWindowCursorState Window, + IReadOnlyCollection PendingDocuments, + IReadOnlyCollection PendingMappings) +{ + public static NvdCursor Empty { get; } = new(TimeWindowCursorState.Empty, Array.Empty(), Array.Empty()); + + public BsonDocument ToBsonDocument() + { + var document = new BsonDocument(); + Window.WriteTo(document); + document["pendingDocuments"] = new BsonArray(PendingDocuments.Select(id => id.ToString())); + document["pendingMappings"] = new BsonArray(PendingMappings.Select(id => id.ToString())); + return document; + } + + public static NvdCursor FromBsonDocument(BsonDocument? document) + { + if (document is null || document.ElementCount == 0) + { + return Empty; + } + + var window = TimeWindowCursorState.FromBsonDocument(document); + var pendingDocuments = ReadGuidArray(document, "pendingDocuments"); + var pendingMappings = ReadGuidArray(document, "pendingMappings"); + + return new NvdCursor(window, pendingDocuments, pendingMappings); + } + + private static IReadOnlyCollection ReadGuidArray(BsonDocument document, string field) + { + if (!document.TryGetValue(field, out var value) || value is not BsonArray array) + { + return Array.Empty(); + } + + var results = new List(array.Count); + foreach (var element in array) + { + if (Guid.TryParse(element.AsString, out var guid)) + { + results.Add(guid); + } + } + + return results; + } + + public NvdCursor WithWindow(TimeWindow window) + => this with { Window = Window.WithWindow(window) }; + + public NvdCursor WithPendingDocuments(IEnumerable ids) + => this with { PendingDocuments = ids?.Distinct().ToArray() ?? Array.Empty() }; + + public NvdCursor WithPendingMappings(IEnumerable ids) + => this with { PendingMappings = ids?.Distinct().ToArray() ?? Array.Empty() }; +} diff --git a/src/StellaOps.Feedser.Source.Nvd/Internal/NvdDiagnostics.cs b/src/StellaOps.Feedser.Source.Nvd/Internal/NvdDiagnostics.cs new file mode 100644 index 00000000..1487b3ab --- /dev/null +++ b/src/StellaOps.Feedser.Source.Nvd/Internal/NvdDiagnostics.cs @@ -0,0 +1,76 @@ +using System.Diagnostics.Metrics; + +namespace StellaOps.Feedser.Source.Nvd.Internal; + +public sealed class NvdDiagnostics : IDisposable +{ + public const string MeterName = "StellaOps.Feedser.Source.Nvd"; + public const string MeterVersion = "1.0.0"; + + private readonly Meter _meter; + private readonly Counter _fetchAttempts; + private readonly Counter _fetchDocuments; + private readonly Counter _fetchFailures; + private readonly Counter _fetchUnchanged; + private readonly Counter _parseSuccess; + private readonly Counter _parseFailures; + private readonly Counter _parseQuarantine; + private readonly Counter _mapSuccess; + + public NvdDiagnostics() + { + _meter = new Meter(MeterName, MeterVersion); + _fetchAttempts = _meter.CreateCounter( + name: "nvd.fetch.attempts", + unit: "operations", + description: "Number of NVD fetch operations attempted, including paginated windows."); + _fetchDocuments = _meter.CreateCounter( + name: "nvd.fetch.documents", + unit: "documents", + description: "Count of NVD documents fetched and persisted."); + _fetchFailures = _meter.CreateCounter( + name: "nvd.fetch.failures", + unit: "operations", + description: "Count of NVD fetch attempts that resulted in an error or missing document."); + _fetchUnchanged = _meter.CreateCounter( + name: "nvd.fetch.unchanged", + unit: "operations", + description: "Count of NVD fetch attempts returning 304 Not Modified."); + _parseSuccess = _meter.CreateCounter( + name: "nvd.parse.success", + unit: "documents", + description: "Count of NVD documents successfully validated and converted into DTOs."); + _parseFailures = _meter.CreateCounter( + name: "nvd.parse.failures", + unit: "documents", + description: "Count of NVD documents that failed parsing due to missing content or read errors."); + _parseQuarantine = _meter.CreateCounter( + name: "nvd.parse.quarantine", + unit: "documents", + description: "Count of NVD documents quarantined due to schema validation failures."); + _mapSuccess = _meter.CreateCounter( + name: "nvd.map.success", + unit: "advisories", + description: "Count of canonical advisories produced by NVD mapping."); + } + + public void FetchAttempt() => _fetchAttempts.Add(1); + + public void FetchDocument() => _fetchDocuments.Add(1); + + public void FetchFailure() => _fetchFailures.Add(1); + + public void FetchUnchanged() => _fetchUnchanged.Add(1); + + public void ParseSuccess() => _parseSuccess.Add(1); + + public void ParseFailure() => _parseFailures.Add(1); + + public void ParseQuarantine() => _parseQuarantine.Add(1); + + public void MapSuccess(long count = 1) => _mapSuccess.Add(count); + + public Meter Meter => _meter; + + public void Dispose() => _meter.Dispose(); +} diff --git a/src/StellaOps.Feedser.Source.Nvd/Internal/NvdMapper.cs b/src/StellaOps.Feedser.Source.Nvd/Internal/NvdMapper.cs new file mode 100644 index 00000000..306ef463 --- /dev/null +++ b/src/StellaOps.Feedser.Source.Nvd/Internal/NvdMapper.cs @@ -0,0 +1,596 @@ +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Text.Json; +using StellaOps.Feedser.Models; +using StellaOps.Feedser.Normalization.Identifiers; +using StellaOps.Feedser.Normalization.Cvss; +using StellaOps.Feedser.Normalization.Text; +using StellaOps.Feedser.Storage.Mongo.Documents; + +namespace StellaOps.Feedser.Source.Nvd.Internal; + +internal static class NvdMapper +{ + public static IReadOnlyList Map(JsonDocument document, DocumentRecord sourceDocument, DateTimeOffset recordedAt) + { + ArgumentNullException.ThrowIfNull(document); + ArgumentNullException.ThrowIfNull(sourceDocument); + + if (!document.RootElement.TryGetProperty("vulnerabilities", out var vulnerabilities) || vulnerabilities.ValueKind != JsonValueKind.Array) + { + return Array.Empty(); + } + + var advisories = new List(vulnerabilities.GetArrayLength()); + var index = 0; + foreach (var vulnerability in vulnerabilities.EnumerateArray()) + { + if (!vulnerability.TryGetProperty("cve", out var cve) || cve.ValueKind != JsonValueKind.Object) + { + index++; + continue; + } + + if (!cve.TryGetProperty("id", out var idElement) || idElement.ValueKind != JsonValueKind.String) + { + index++; + continue; + } + + var cveId = idElement.GetString(); + var advisoryKey = string.IsNullOrWhiteSpace(cveId) + ? $"nvd:{sourceDocument.Id:N}:{index}" + : cveId; + + var published = TryGetDateTime(cve, "published"); + var modified = TryGetDateTime(cve, "lastModified"); + var description = GetNormalizedDescription(cve); + + var references = GetReferences(cve, sourceDocument, recordedAt); + var affectedPackages = GetAffectedPackages(cve, sourceDocument, recordedAt); + var cvssMetrics = GetCvssMetrics(cve, sourceDocument, recordedAt, out var severity); + + var provenance = new[] + { + new AdvisoryProvenance( + NvdConnectorPlugin.SourceName, + "document", + sourceDocument.Uri, + sourceDocument.FetchedAt, + new[] { ProvenanceFieldMasks.Advisory }), + new AdvisoryProvenance( + NvdConnectorPlugin.SourceName, + "mapping", + string.IsNullOrWhiteSpace(cveId) ? advisoryKey : cveId, + recordedAt, + new[] { ProvenanceFieldMasks.Advisory }), + }; + + var title = string.IsNullOrWhiteSpace(cveId) ? advisoryKey : cveId; + + var aliasCandidates = new List(capacity: 2); + if (!string.IsNullOrWhiteSpace(cveId)) + { + aliasCandidates.Add(cveId); + } + + aliasCandidates.Add(advisoryKey); + + var advisory = new Advisory( + advisoryKey: advisoryKey, + title: title, + summary: string.IsNullOrEmpty(description.Text) ? null : description.Text, + language: description.Language, + published: published, + modified: modified, + severity: severity, + exploitKnown: false, + aliases: aliasCandidates, + references: references, + affectedPackages: affectedPackages, + cvssMetrics: cvssMetrics, + provenance: provenance); + + advisories.Add(advisory); + index++; + } + + return advisories; + } + + private static NormalizedDescription GetNormalizedDescription(JsonElement cve) + { + var candidates = new List(); + + if (cve.TryGetProperty("descriptions", out var descriptions) && descriptions.ValueKind == JsonValueKind.Array) + { + foreach (var item in descriptions.EnumerateArray()) + { + if (item.ValueKind != JsonValueKind.Object) + { + continue; + } + + var text = item.TryGetProperty("value", out var valueElement) && valueElement.ValueKind == JsonValueKind.String + ? valueElement.GetString() + : null; + var lang = item.TryGetProperty("lang", out var langElement) && langElement.ValueKind == JsonValueKind.String + ? langElement.GetString() + : null; + + if (!string.IsNullOrWhiteSpace(text)) + { + candidates.Add(new LocalizedText(text, lang)); + } + } + } + + return DescriptionNormalizer.Normalize(candidates); + } + + private static DateTimeOffset? TryGetDateTime(JsonElement element, string propertyName) + { + if (!element.TryGetProperty(propertyName, out var property) || property.ValueKind != JsonValueKind.String) + { + return null; + } + + return DateTimeOffset.TryParse(property.GetString(), out var parsed) ? parsed : null; + } + + private static IReadOnlyList GetReferences(JsonElement cve, DocumentRecord document, DateTimeOffset recordedAt) + { + var references = new List(); + if (!cve.TryGetProperty("references", out var referencesElement) || referencesElement.ValueKind != JsonValueKind.Array) + { + return references; + } + + foreach (var reference in referencesElement.EnumerateArray()) + { + if (!reference.TryGetProperty("url", out var urlElement) || urlElement.ValueKind != JsonValueKind.String) + { + continue; + } + + var url = urlElement.GetString(); + if (string.IsNullOrWhiteSpace(url) || !Validation.LooksLikeHttpUrl(url)) + { + continue; + } + + var sourceTag = reference.TryGetProperty("source", out var sourceElement) ? sourceElement.GetString() : null; + string? kind = null; + if (reference.TryGetProperty("tags", out var tagsElement) && tagsElement.ValueKind == JsonValueKind.Array) + { + kind = tagsElement.EnumerateArray().Select(static t => t.GetString()).FirstOrDefault(static tag => !string.IsNullOrWhiteSpace(tag))?.ToLowerInvariant(); + } + + references.Add(new AdvisoryReference( + url: url, + kind: kind, + sourceTag: sourceTag, + summary: null, + provenance: new AdvisoryProvenance( + NvdConnectorPlugin.SourceName, + "reference", + url, + recordedAt, + new[] { ProvenanceFieldMasks.References }))); + } + + AppendWeaknessReferences(cve, references, recordedAt); + return references; + } + + private static void AppendWeaknessReferences(JsonElement cve, List references, DateTimeOffset recordedAt) + { + if (!cve.TryGetProperty("weaknesses", out var weaknesses) || weaknesses.ValueKind != JsonValueKind.Array) + { + return; + } + + var existing = new HashSet(references.Select(reference => reference.Url), StringComparer.OrdinalIgnoreCase); + + foreach (var weakness in weaknesses.EnumerateArray()) + { + if (!weakness.TryGetProperty("description", out var descriptions) || descriptions.ValueKind != JsonValueKind.Array) + { + continue; + } + + string? cweId = null; + string? name = null; + + foreach (var description in descriptions.EnumerateArray()) + { + if (description.ValueKind != JsonValueKind.Object) + { + continue; + } + + if (!description.TryGetProperty("value", out var valueElement) || valueElement.ValueKind != JsonValueKind.String) + { + continue; + } + + var value = valueElement.GetString(); + if (string.IsNullOrWhiteSpace(value)) + { + continue; + } + + var trimmed = value.Trim(); + if (trimmed.StartsWith("CWE-", StringComparison.OrdinalIgnoreCase)) + { + cweId ??= trimmed.ToUpperInvariant(); + } + else + { + name ??= trimmed; + } + } + + if (string.IsNullOrWhiteSpace(cweId)) + { + continue; + } + + var url = BuildCweUrl(cweId); + if (url is null || existing.Contains(url)) + { + continue; + } + + var provenance = new AdvisoryProvenance( + NvdConnectorPlugin.SourceName, + "reference", + url, + recordedAt, + new[] { ProvenanceFieldMasks.References }); + + references.Add(new AdvisoryReference(url, "weakness", cweId, name, provenance)); + existing.Add(url); + } + } + + private static IReadOnlyList GetAffectedPackages(JsonElement cve, DocumentRecord document, DateTimeOffset recordedAt) + { + var packages = new Dictionary(StringComparer.Ordinal); + if (!cve.TryGetProperty("configurations", out var configurations) || configurations.ValueKind != JsonValueKind.Object) + { + return Array.Empty(); + } + + if (!configurations.TryGetProperty("nodes", out var nodes) || nodes.ValueKind != JsonValueKind.Array) + { + return Array.Empty(); + } + + foreach (var node in nodes.EnumerateArray()) + { + if (!node.TryGetProperty("cpeMatch", out var matches) || matches.ValueKind != JsonValueKind.Array) + { + continue; + } + + foreach (var match in matches.EnumerateArray()) + { + if (match.TryGetProperty("vulnerable", out var vulnerableElement) && vulnerableElement.ValueKind == JsonValueKind.False) + { + continue; + } + + if (!match.TryGetProperty("criteria", out var criteriaElement) || criteriaElement.ValueKind != JsonValueKind.String) + { + continue; + } + + var criteria = criteriaElement.GetString(); + if (string.IsNullOrWhiteSpace(criteria)) + { + continue; + } + + var identifier = IdentifierNormalizer.TryNormalizeCpe(criteria, out var normalizedCpe) && !string.IsNullOrWhiteSpace(normalizedCpe) + ? normalizedCpe + : criteria.Trim(); + + var provenance = new AdvisoryProvenance( + NvdConnectorPlugin.SourceName, + "cpe", + document.Uri, + recordedAt, + new[] { ProvenanceFieldMasks.AffectedPackages }); + if (!packages.TryGetValue(identifier, out var accumulator)) + { + accumulator = new PackageAccumulator(); + packages[identifier] = accumulator; + } + + var range = BuildVersionRange(match, criteria, provenance); + if (range is not null) + { + accumulator.Ranges.Add(range); + } + + accumulator.Provenance.Add(provenance); + } + } + + if (packages.Count == 0) + { + return Array.Empty(); + } + + return packages + .OrderBy(static kvp => kvp.Key, StringComparer.Ordinal) + .Select(static kvp => + { + var ranges = kvp.Value.Ranges.Count == 0 + ? Array.Empty() + : kvp.Value.Ranges + .OrderBy(static range => range, AffectedVersionRangeComparer.Instance) + .ToArray(); + + var provenance = kvp.Value.Provenance + .OrderBy(static p => p.Source, StringComparer.Ordinal) + .ThenBy(static p => p.Kind, StringComparer.Ordinal) + .ThenBy(static p => p.Value, StringComparer.Ordinal) + .ThenBy(static p => p.RecordedAt.UtcDateTime) + .ToArray(); + + return new AffectedPackage( + type: AffectedPackageTypes.Cpe, + identifier: kvp.Key, + platform: null, + versionRanges: ranges, + statuses: Array.Empty(), + provenance: provenance); + }) + .ToArray(); + } + + private static IReadOnlyList GetCvssMetrics(JsonElement cve, DocumentRecord document, DateTimeOffset recordedAt, out string? severity) + { + severity = null; + if (!cve.TryGetProperty("metrics", out var metrics) || metrics.ValueKind != JsonValueKind.Object) + { + return Array.Empty(); + } + + var sources = new[] { "cvssMetricV31", "cvssMetricV30", "cvssMetricV2" }; + foreach (var source in sources) + { + if (!metrics.TryGetProperty(source, out var array) || array.ValueKind != JsonValueKind.Array) + { + continue; + } + + var list = new List(); + foreach (var item in array.EnumerateArray()) + { + if (!item.TryGetProperty("cvssData", out var data) || data.ValueKind != JsonValueKind.Object) + { + continue; + } + + if (!data.TryGetProperty("vectorString", out var vectorElement) || vectorElement.ValueKind != JsonValueKind.String) + { + continue; + } + + if (!data.TryGetProperty("baseScore", out var scoreElement) || scoreElement.ValueKind != JsonValueKind.Number) + { + continue; + } + + if (!data.TryGetProperty("baseSeverity", out var severityElement) || severityElement.ValueKind != JsonValueKind.String) + { + continue; + } + + var vector = vectorElement.GetString() ?? string.Empty; + var baseScore = scoreElement.GetDouble(); + var baseSeverity = severityElement.GetString(); + var versionToken = source switch + { + "cvssMetricV30" => "3.0", + "cvssMetricV31" => "3.1", + _ => "2.0", + }; + + if (!CvssMetricNormalizer.TryNormalize(versionToken, vector, baseScore, baseSeverity, out var normalized)) + { + continue; + } + + severity ??= normalized.BaseSeverity; + + list.Add(normalized.ToModel(new AdvisoryProvenance( + NvdConnectorPlugin.SourceName, + "cvss", + normalized.Vector, + recordedAt, + new[] { ProvenanceFieldMasks.CvssMetrics }))); + } + + if (list.Count > 0) + { + return list; + } + } + + return Array.Empty(); + } + + private static AffectedVersionRange? BuildVersionRange(JsonElement match, string criteria, AdvisoryProvenance provenance) + { + static string? ReadString(JsonElement parent, string property) + { + if (!parent.TryGetProperty(property, out var value) || value.ValueKind != JsonValueKind.String) + { + return null; + } + + var text = value.GetString(); + return string.IsNullOrWhiteSpace(text) ? null : text.Trim(); + } + + var version = ReadString(match, "version"); + if (string.Equals(version, "*", StringComparison.Ordinal)) + { + version = null; + } + + version ??= TryExtractVersionFromCriteria(criteria); + + var versionStartIncluding = ReadString(match, "versionStartIncluding"); + var versionStartExcluding = ReadString(match, "versionStartExcluding"); + var versionEndIncluding = ReadString(match, "versionEndIncluding"); + var versionEndExcluding = ReadString(match, "versionEndExcluding"); + + var vendorExtensions = new Dictionary(StringComparer.Ordinal); + if (versionStartIncluding is not null) + { + vendorExtensions["versionStartIncluding"] = versionStartIncluding; + } + + if (versionStartExcluding is not null) + { + vendorExtensions["versionStartExcluding"] = versionStartExcluding; + } + + if (versionEndIncluding is not null) + { + vendorExtensions["versionEndIncluding"] = versionEndIncluding; + } + + if (versionEndExcluding is not null) + { + vendorExtensions["versionEndExcluding"] = versionEndExcluding; + } + + if (version is not null) + { + vendorExtensions["version"] = version; + } + + string? introduced = null; + string? fixedVersion = null; + string? lastAffected = null; + var expressionParts = new List(); + + if (versionStartIncluding is not null) + { + introduced = versionStartIncluding; + expressionParts.Add($">={versionStartIncluding}"); + } + + if (versionStartExcluding is not null) + { + introduced ??= versionStartExcluding; + expressionParts.Add($">{versionStartExcluding}"); + } + + if (versionEndExcluding is not null) + { + fixedVersion = versionEndExcluding; + expressionParts.Add($"<{versionEndExcluding}"); + } + + if (versionEndIncluding is not null) + { + lastAffected = versionEndIncluding; + expressionParts.Add($"<={versionEndIncluding}"); + } + + if (version is not null) + { + introduced ??= version; + lastAffected ??= version; + expressionParts.Add($"=={version}"); + } + + if (introduced is null && fixedVersion is null && lastAffected is null && vendorExtensions.Count == 0) + { + return null; + } + + var rangeExpression = expressionParts.Count > 0 ? string.Join(' ', expressionParts) : null; + IReadOnlyDictionary? extensions = vendorExtensions.Count == 0 ? null : vendorExtensions; + var primitives = extensions is null ? null : new RangePrimitives(null, null, null, extensions); + + var rangeProvenance = new AdvisoryProvenance( + provenance.Source, + provenance.Kind, + provenance.Value, + provenance.RecordedAt, + new[] { ProvenanceFieldMasks.VersionRanges }); + + return new AffectedVersionRange( + rangeKind: "cpe", + introducedVersion: introduced, + fixedVersion: fixedVersion, + lastAffectedVersion: lastAffected, + rangeExpression: rangeExpression, + provenance: rangeProvenance, + primitives); + } + + private static string? BuildCweUrl(string cweId) + { + var dashIndex = cweId.IndexOf('-'); + if (dashIndex < 0 || dashIndex == cweId.Length - 1) + { + return null; + } + + var digits = new StringBuilder(); + for (var i = dashIndex + 1; i < cweId.Length; i++) + { + var ch = cweId[i]; + if (char.IsDigit(ch)) + { + digits.Append(ch); + } + } + + return digits.Length == 0 ? null : $"https://cwe.mitre.org/data/definitions/{digits}.html"; + } + + private static string? TryExtractVersionFromCriteria(string criteria) + { + if (string.IsNullOrWhiteSpace(criteria)) + { + return null; + } + + var segments = criteria.Split(':'); + if (segments.Length < 6) + { + return null; + } + + var version = segments[5]; + if (string.IsNullOrWhiteSpace(version)) + { + return null; + } + + if (string.Equals(version, "*", StringComparison.Ordinal) || string.Equals(version, "-", StringComparison.Ordinal)) + { + return null; + } + + return version; + } + + private sealed class PackageAccumulator + { + public List Ranges { get; } = new(); + + public List Provenance { get; } = new(); + } +} diff --git a/src/StellaOps.Feedser.Source.Nvd/Internal/NvdSchemaProvider.cs b/src/StellaOps.Feedser.Source.Nvd/Internal/NvdSchemaProvider.cs new file mode 100644 index 00000000..e7e9e54f --- /dev/null +++ b/src/StellaOps.Feedser.Source.Nvd/Internal/NvdSchemaProvider.cs @@ -0,0 +1,25 @@ +using System.IO; +using System.Reflection; +using System.Threading; +using Json.Schema; + +namespace StellaOps.Feedser.Source.Nvd.Internal; + +internal static class NvdSchemaProvider +{ + private static readonly Lazy Cached = new(LoadSchema, LazyThreadSafetyMode.ExecutionAndPublication); + + public static JsonSchema Schema => Cached.Value; + + private static JsonSchema LoadSchema() + { + var assembly = typeof(NvdSchemaProvider).GetTypeInfo().Assembly; + const string resourceName = "StellaOps.Feedser.Source.Nvd.Schemas.nvd-vulnerability.schema.json"; + + using var stream = assembly.GetManifestResourceStream(resourceName) + ?? throw new InvalidOperationException($"Embedded schema '{resourceName}' not found."); + using var reader = new StreamReader(stream); + var schemaText = reader.ReadToEnd(); + return JsonSchema.FromText(schemaText); + } +} diff --git a/src/StellaOps.Feedser.Source.Nvd/NvdConnector.cs b/src/StellaOps.Feedser.Source.Nvd/NvdConnector.cs new file mode 100644 index 00000000..f315b3ef --- /dev/null +++ b/src/StellaOps.Feedser.Source.Nvd/NvdConnector.cs @@ -0,0 +1,565 @@ +using System.Globalization; +using System.Security.Cryptography; +using System.Text; +using System.Text.Json; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Options; +using MongoDB.Bson; +using StellaOps.Feedser.Models; +using StellaOps.Feedser.Source.Common; +using StellaOps.Feedser.Source.Common.Fetch; +using StellaOps.Feedser.Source.Common.Json; +using StellaOps.Feedser.Source.Common.Cursors; +using StellaOps.Feedser.Source.Nvd.Configuration; +using StellaOps.Feedser.Source.Nvd.Internal; +using StellaOps.Feedser.Storage.Mongo; +using StellaOps.Feedser.Storage.Mongo.Advisories; +using StellaOps.Feedser.Storage.Mongo.Documents; +using StellaOps.Feedser.Storage.Mongo.Dtos; +using StellaOps.Feedser.Storage.Mongo.ChangeHistory; +using StellaOps.Plugin; +using Json.Schema; + +namespace StellaOps.Feedser.Source.Nvd; + +public sealed class NvdConnector : IFeedConnector +{ + private readonly SourceFetchService _fetchService; + private readonly RawDocumentStorage _rawDocumentStorage; + private readonly IDocumentStore _documentStore; + private readonly IDtoStore _dtoStore; + private readonly IAdvisoryStore _advisoryStore; + private readonly IChangeHistoryStore _changeHistoryStore; + private readonly ISourceStateRepository _stateRepository; + private readonly IJsonSchemaValidator _schemaValidator; + private readonly NvdOptions _options; + private readonly TimeProvider _timeProvider; + private readonly ILogger _logger; + private readonly NvdDiagnostics _diagnostics; + + private static readonly JsonSchema Schema = NvdSchemaProvider.Schema; + + public NvdConnector( + SourceFetchService fetchService, + RawDocumentStorage rawDocumentStorage, + IDocumentStore documentStore, + IDtoStore dtoStore, + IAdvisoryStore advisoryStore, + IChangeHistoryStore changeHistoryStore, + ISourceStateRepository stateRepository, + IJsonSchemaValidator schemaValidator, + IOptions options, + NvdDiagnostics diagnostics, + TimeProvider? timeProvider, + ILogger logger) + { + _fetchService = fetchService ?? throw new ArgumentNullException(nameof(fetchService)); + _rawDocumentStorage = rawDocumentStorage ?? throw new ArgumentNullException(nameof(rawDocumentStorage)); + _documentStore = documentStore ?? throw new ArgumentNullException(nameof(documentStore)); + _dtoStore = dtoStore ?? throw new ArgumentNullException(nameof(dtoStore)); + _advisoryStore = advisoryStore ?? throw new ArgumentNullException(nameof(advisoryStore)); + _changeHistoryStore = changeHistoryStore ?? throw new ArgumentNullException(nameof(changeHistoryStore)); + _stateRepository = stateRepository ?? throw new ArgumentNullException(nameof(stateRepository)); + _schemaValidator = schemaValidator ?? throw new ArgumentNullException(nameof(schemaValidator)); + _options = options?.Value ?? throw new ArgumentNullException(nameof(options)); + _options.Validate(); + _diagnostics = diagnostics ?? throw new ArgumentNullException(nameof(diagnostics)); + _timeProvider = timeProvider ?? TimeProvider.System; + _logger = logger ?? throw new ArgumentNullException(nameof(logger)); + } + + public string SourceName => NvdConnectorPlugin.SourceName; + + public async Task FetchAsync(IServiceProvider services, CancellationToken cancellationToken) + { + var cursor = await GetCursorAsync(cancellationToken).ConfigureAwait(false); + var now = _timeProvider.GetUtcNow(); + + var windowOptions = new TimeWindowCursorOptions + { + WindowSize = _options.WindowSize, + Overlap = _options.WindowOverlap, + InitialBackfill = _options.InitialBackfill, + }; + + var window = TimeWindowCursorPlanner.GetNextWindow(now, cursor.Window, windowOptions); + var requestUri = BuildRequestUri(window); + + var metadata = new Dictionary(StringComparer.Ordinal) + { + ["windowStart"] = window.Start.ToString("O"), + ["windowEnd"] = window.End.ToString("O"), + }; + metadata["startIndex"] = "0"; + + try + { + _diagnostics.FetchAttempt(); + + var result = await _fetchService.FetchAsync( + new SourceFetchRequest( + NvdOptions.HttpClientName, + SourceName, + requestUri) + { + Metadata = metadata + }, + cancellationToken).ConfigureAwait(false); + + if (result.IsNotModified) + { + _diagnostics.FetchUnchanged(); + _logger.LogDebug("NVD window {Start} - {End} returned 304", window.Start, window.End); + await UpdateCursorAsync(cursor.WithWindow(window), cancellationToken).ConfigureAwait(false); + return; + } + + if (!result.IsSuccess || result.Document is null) + { + _diagnostics.FetchFailure(); + return; + } + + _diagnostics.FetchDocument(); + + var pendingDocuments = new HashSet(cursor.PendingDocuments) + { + result.Document.Id + }; + + var additionalDocuments = await FetchAdditionalPagesAsync( + window, + metadata, + result.Document, + cancellationToken).ConfigureAwait(false); + + foreach (var documentId in additionalDocuments) + { + pendingDocuments.Add(documentId); + } + + var updated = cursor + .WithWindow(window) + .WithPendingDocuments(pendingDocuments) + .WithPendingMappings(cursor.PendingMappings); + + await UpdateCursorAsync(updated, cancellationToken).ConfigureAwait(false); + } + catch (Exception ex) + { + _diagnostics.FetchFailure(); + _logger.LogError(ex, "NVD fetch failed for {Uri}", requestUri); + await _stateRepository.MarkFailureAsync(SourceName, now, TimeSpan.FromMinutes(5), ex.Message, cancellationToken).ConfigureAwait(false); + throw; + } + } + + public async Task ParseAsync(IServiceProvider services, CancellationToken cancellationToken) + { + var cursor = await GetCursorAsync(cancellationToken).ConfigureAwait(false); + if (cursor.PendingDocuments.Count == 0) + { + return; + } + + var remainingFetch = cursor.PendingDocuments.ToList(); + var pendingMapping = cursor.PendingMappings.ToList(); + + foreach (var documentId in cursor.PendingDocuments) + { + var document = await _documentStore.FindAsync(documentId, cancellationToken).ConfigureAwait(false); + if (document is null) + { + _diagnostics.ParseFailure(); + remainingFetch.Remove(documentId); + pendingMapping.Remove(documentId); + continue; + } + + if (!document.GridFsId.HasValue) + { + _logger.LogWarning("Document {DocumentId} is missing GridFS content; skipping", documentId); + _diagnostics.ParseFailure(); + remainingFetch.Remove(documentId); + pendingMapping.Remove(documentId); + continue; + } + + var rawBytes = await _rawDocumentStorage.DownloadAsync(document.GridFsId.Value, cancellationToken).ConfigureAwait(false); + try + { + using var jsonDocument = JsonDocument.Parse(rawBytes); + try + { + _schemaValidator.Validate(jsonDocument, Schema, document.Uri); + } + catch (JsonSchemaValidationException ex) + { + _logger.LogWarning(ex, "NVD schema validation failed for document {DocumentId} ({Uri})", document.Id, document.Uri); + await _documentStore.UpdateStatusAsync(document.Id, DocumentStatuses.Failed, cancellationToken).ConfigureAwait(false); + remainingFetch.Remove(documentId); + pendingMapping.Remove(documentId); + _diagnostics.ParseQuarantine(); + continue; + } + + var sanitized = JsonSerializer.Serialize(jsonDocument.RootElement); + var payload = BsonDocument.Parse(sanitized); + + var dtoRecord = new DtoRecord( + Guid.NewGuid(), + document.Id, + SourceName, + "nvd.cve.v2", + payload, + _timeProvider.GetUtcNow()); + + await _dtoStore.UpsertAsync(dtoRecord, cancellationToken).ConfigureAwait(false); + await _documentStore.UpdateStatusAsync(document.Id, DocumentStatuses.PendingMap, cancellationToken).ConfigureAwait(false); + _diagnostics.ParseSuccess(); + + remainingFetch.Remove(documentId); + if (!pendingMapping.Contains(documentId)) + { + pendingMapping.Add(documentId); + } + } + catch (JsonException ex) + { + _logger.LogWarning(ex, "Failed to parse NVD JSON payload for document {DocumentId} ({Uri})", document.Id, document.Uri); + await _documentStore.UpdateStatusAsync(document.Id, DocumentStatuses.Failed, cancellationToken).ConfigureAwait(false); + remainingFetch.Remove(documentId); + pendingMapping.Remove(documentId); + _diagnostics.ParseFailure(); + } + } + + var updatedCursor = cursor + .WithPendingDocuments(remainingFetch) + .WithPendingMappings(pendingMapping); + + await UpdateCursorAsync(updatedCursor, cancellationToken).ConfigureAwait(false); + } + + public async Task MapAsync(IServiceProvider services, CancellationToken cancellationToken) + { + var cursor = await GetCursorAsync(cancellationToken).ConfigureAwait(false); + if (cursor.PendingMappings.Count == 0) + { + return; + } + + var pendingMapping = cursor.PendingMappings.ToList(); + var now = _timeProvider.GetUtcNow(); + + foreach (var documentId in cursor.PendingMappings) + { + var dto = await _dtoStore.FindByDocumentIdAsync(documentId, cancellationToken).ConfigureAwait(false); + var document = await _documentStore.FindAsync(documentId, cancellationToken).ConfigureAwait(false); + + if (dto is null || document is null) + { + pendingMapping.Remove(documentId); + continue; + } + + var json = dto.Payload.ToJson(new MongoDB.Bson.IO.JsonWriterSettings + { + OutputMode = MongoDB.Bson.IO.JsonOutputMode.RelaxedExtendedJson, + }); + + using var jsonDocument = JsonDocument.Parse(json); + var advisories = NvdMapper.Map(jsonDocument, document, now) + .GroupBy(static advisory => advisory.AdvisoryKey, StringComparer.Ordinal) + .Select(static group => group.First()) + .ToArray(); + + var mappedCount = 0L; + foreach (var advisory in advisories) + { + if (string.IsNullOrWhiteSpace(advisory.AdvisoryKey)) + { + _logger.LogWarning("Skipping advisory with missing key for document {DocumentId} ({Uri})", document.Id, document.Uri); + continue; + } + + var previous = await _advisoryStore.FindAsync(advisory.AdvisoryKey, cancellationToken).ConfigureAwait(false); + await _advisoryStore.UpsertAsync(advisory, cancellationToken).ConfigureAwait(false); + if (previous is not null) + { + await RecordChangeHistoryAsync(advisory, previous, document, now, cancellationToken).ConfigureAwait(false); + } + mappedCount++; + } + + if (mappedCount > 0) + { + _diagnostics.MapSuccess(mappedCount); + } + + await _documentStore.UpdateStatusAsync(document.Id, DocumentStatuses.Mapped, cancellationToken).ConfigureAwait(false); + pendingMapping.Remove(documentId); + } + + var updatedCursor = cursor.WithPendingMappings(pendingMapping); + await UpdateCursorAsync(updatedCursor, cancellationToken).ConfigureAwait(false); + } + + private async Task> FetchAdditionalPagesAsync( + TimeWindow window, + IReadOnlyDictionary baseMetadata, + DocumentRecord firstDocument, + CancellationToken cancellationToken) + { + if (firstDocument.GridFsId is null) + { + return Array.Empty(); + } + + byte[] rawBytes; + try + { + rawBytes = await _rawDocumentStorage.DownloadAsync(firstDocument.GridFsId.Value, cancellationToken).ConfigureAwait(false); + } + catch (Exception ex) + { + _logger.LogWarning(ex, "Unable to download NVD first page {DocumentId} to evaluate pagination", firstDocument.Id); + return Array.Empty(); + } + + try + { + using var jsonDocument = JsonDocument.Parse(rawBytes); + var root = jsonDocument.RootElement; + + if (!TryReadInt32(root, "totalResults", out var totalResults) || !TryReadInt32(root, "resultsPerPage", out var resultsPerPage)) + { + return Array.Empty(); + } + + if (resultsPerPage <= 0 || totalResults <= resultsPerPage) + { + return Array.Empty(); + } + + var fetchedDocuments = new List(); + + foreach (var startIndex in PaginationPlanner.EnumerateAdditionalPages(totalResults, resultsPerPage)) + { + var metadata = new Dictionary(StringComparer.Ordinal); + foreach (var kvp in baseMetadata) + { + metadata[kvp.Key] = kvp.Value; + } + metadata["startIndex"] = startIndex.ToString(CultureInfo.InvariantCulture); + + var request = new SourceFetchRequest( + NvdOptions.HttpClientName, + SourceName, + BuildRequestUri(window, startIndex)) + { + Metadata = metadata + }; + + SourceFetchResult pageResult; + try + { + _diagnostics.FetchAttempt(); + pageResult = await _fetchService.FetchAsync(request, cancellationToken).ConfigureAwait(false); + } + catch (Exception ex) + { + _diagnostics.FetchFailure(); + _logger.LogError(ex, "NVD fetch failed for page starting at {StartIndex}", startIndex); + throw; + } + + if (pageResult.IsNotModified) + { + _diagnostics.FetchUnchanged(); + continue; + } + + if (!pageResult.IsSuccess || pageResult.Document is null) + { + _diagnostics.FetchFailure(); + _logger.LogWarning("NVD fetch for page starting at {StartIndex} returned status {Status}", startIndex, pageResult.StatusCode); + continue; + } + + _diagnostics.FetchDocument(); + fetchedDocuments.Add(pageResult.Document.Id); + } + + return fetchedDocuments; + } + catch (JsonException ex) + { + _logger.LogWarning(ex, "Failed to parse NVD first page {DocumentId} while determining pagination", firstDocument.Id); + return Array.Empty(); + } + } + + private static bool TryReadInt32(JsonElement root, string propertyName, out int value) + { + value = 0; + if (!root.TryGetProperty(propertyName, out var property) || property.ValueKind != JsonValueKind.Number) + { + return false; + } + + if (property.TryGetInt32(out var intValue)) + { + value = intValue; + return true; + } + + if (property.TryGetInt64(out var longValue)) + { + if (longValue > int.MaxValue) + { + value = int.MaxValue; + return true; + } + + value = (int)longValue; + return true; + } + + return false; + } + + private async Task RecordChangeHistoryAsync( + Advisory current, + Advisory previous, + DocumentRecord document, + DateTimeOffset capturedAt, + CancellationToken cancellationToken) + { + if (current.Equals(previous)) + { + return; + } + + var currentSnapshot = SnapshotSerializer.ToSnapshot(current); + var previousSnapshot = SnapshotSerializer.ToSnapshot(previous); + + if (string.Equals(currentSnapshot, previousSnapshot, StringComparison.Ordinal)) + { + return; + } + + var changes = ComputeChanges(previousSnapshot, currentSnapshot); + if (changes.Count == 0) + { + return; + } + + var documentHash = string.IsNullOrWhiteSpace(document.Sha256) + ? ComputeHash(currentSnapshot) + : document.Sha256; + + var record = new ChangeHistoryRecord( + Guid.NewGuid(), + SourceName, + current.AdvisoryKey, + document.Id, + documentHash, + ComputeHash(currentSnapshot), + ComputeHash(previousSnapshot), + currentSnapshot, + previousSnapshot, + changes, + capturedAt); + + await _changeHistoryStore.AddAsync(record, cancellationToken).ConfigureAwait(false); + } + + private static IReadOnlyList ComputeChanges(string previousSnapshot, string currentSnapshot) + { + using var previousDocument = JsonDocument.Parse(previousSnapshot); + using var currentDocument = JsonDocument.Parse(currentSnapshot); + + var previousRoot = previousDocument.RootElement; + var currentRoot = currentDocument.RootElement; + var fields = new HashSet(StringComparer.Ordinal); + + foreach (var property in previousRoot.EnumerateObject()) + { + fields.Add(property.Name); + } + + foreach (var property in currentRoot.EnumerateObject()) + { + fields.Add(property.Name); + } + + var changes = new List(); + foreach (var field in fields.OrderBy(static name => name, StringComparer.Ordinal)) + { + var hasPrevious = previousRoot.TryGetProperty(field, out var previousValue); + var hasCurrent = currentRoot.TryGetProperty(field, out var currentValue); + + if (!hasPrevious && hasCurrent) + { + changes.Add(new ChangeHistoryFieldChange(field, "Added", null, SerializeElement(currentValue))); + continue; + } + + if (hasPrevious && !hasCurrent) + { + changes.Add(new ChangeHistoryFieldChange(field, "Removed", SerializeElement(previousValue), null)); + continue; + } + + if (hasPrevious && hasCurrent && !JsonElement.DeepEquals(previousValue, currentValue)) + { + changes.Add(new ChangeHistoryFieldChange(field, "Modified", SerializeElement(previousValue), SerializeElement(currentValue))); + } + } + + return changes; + } + + private static string SerializeElement(JsonElement element) + => JsonSerializer.Serialize(element, new JsonSerializerOptions { WriteIndented = false }); + + private static string ComputeHash(string snapshot) + { + var bytes = Encoding.UTF8.GetBytes(snapshot); + var hash = SHA256.HashData(bytes); + return $"sha256:{Convert.ToHexString(hash).ToLowerInvariant()}"; + } + + private async Task GetCursorAsync(CancellationToken cancellationToken) + { + var record = await _stateRepository.TryGetAsync(SourceName, cancellationToken).ConfigureAwait(false); + return NvdCursor.FromBsonDocument(record?.Cursor); + } + + private async Task UpdateCursorAsync(NvdCursor cursor, CancellationToken cancellationToken) + { + var completedAt = _timeProvider.GetUtcNow(); + await _stateRepository.UpdateCursorAsync(SourceName, cursor.ToBsonDocument(), completedAt, cancellationToken).ConfigureAwait(false); + } + + private Uri BuildRequestUri(TimeWindow window, int startIndex = 0) + { + var builder = new UriBuilder(_options.BaseEndpoint); + + var parameters = new Dictionary + { + ["lastModifiedStartDate"] = window.Start.ToString("yyyy-MM-dd'T'HH:mm:ss.fffK"), + ["lastModifiedEndDate"] = window.End.ToString("yyyy-MM-dd'T'HH:mm:ss.fffK"), + ["resultsPerPage"] = "2000", + }; + + if (startIndex > 0) + { + parameters["startIndex"] = startIndex.ToString(CultureInfo.InvariantCulture); + } + + builder.Query = string.Join("&", parameters.Select(static kvp => $"{System.Net.WebUtility.UrlEncode(kvp.Key)}={System.Net.WebUtility.UrlEncode(kvp.Value)}")); + return builder.Uri; + } +} diff --git a/src/StellaOps.Feedser.Source.Nvd/NvdConnectorPlugin.cs b/src/StellaOps.Feedser.Source.Nvd/NvdConnectorPlugin.cs new file mode 100644 index 00000000..27be1e07 --- /dev/null +++ b/src/StellaOps.Feedser.Source.Nvd/NvdConnectorPlugin.cs @@ -0,0 +1,19 @@ +using Microsoft.Extensions.DependencyInjection; +using StellaOps.Plugin; + +namespace StellaOps.Feedser.Source.Nvd; + +public sealed class NvdConnectorPlugin : IConnectorPlugin +{ + public string Name => SourceName; + + public static string SourceName => "nvd"; + + public bool IsAvailable(IServiceProvider services) => services is not null; + + public IFeedConnector Create(IServiceProvider services) + { + ArgumentNullException.ThrowIfNull(services); + return ActivatorUtilities.CreateInstance(services); + } +} diff --git a/src/StellaOps.Feedser.Source.Nvd/NvdServiceCollectionExtensions.cs b/src/StellaOps.Feedser.Source.Nvd/NvdServiceCollectionExtensions.cs new file mode 100644 index 00000000..f6d1620e --- /dev/null +++ b/src/StellaOps.Feedser.Source.Nvd/NvdServiceCollectionExtensions.cs @@ -0,0 +1,35 @@ +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Options; +using StellaOps.Feedser.Source.Common.Http; +using StellaOps.Feedser.Source.Nvd.Configuration; +using StellaOps.Feedser.Source.Nvd.Internal; + +namespace StellaOps.Feedser.Source.Nvd; + +public static class NvdServiceCollectionExtensions +{ + public static IServiceCollection AddNvdConnector(this IServiceCollection services, Action configure) + { + ArgumentNullException.ThrowIfNull(services); + ArgumentNullException.ThrowIfNull(configure); + + services.AddOptions() + .Configure(configure) + .PostConfigure(static opts => opts.Validate()); + + services.AddSourceHttpClient(NvdOptions.HttpClientName, (sp, clientOptions) => + { + var options = sp.GetRequiredService>().Value; + clientOptions.BaseAddress = options.BaseEndpoint; + clientOptions.Timeout = TimeSpan.FromSeconds(30); + clientOptions.UserAgent = "StellaOps.Feedser.Nvd/1.0"; + clientOptions.AllowedHosts.Clear(); + clientOptions.AllowedHosts.Add(options.BaseEndpoint.Host); + clientOptions.DefaultRequestHeaders["Accept"] = "application/json"; + }); + + services.AddSingleton(); + services.AddTransient(); + return services; + } +} diff --git a/src/StellaOps.Feedser.Source.Nvd/Schemas/nvd-vulnerability.schema.json b/src/StellaOps.Feedser.Source.Nvd/Schemas/nvd-vulnerability.schema.json new file mode 100644 index 00000000..31ebafca --- /dev/null +++ b/src/StellaOps.Feedser.Source.Nvd/Schemas/nvd-vulnerability.schema.json @@ -0,0 +1,115 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "type": "object", + "required": ["vulnerabilities"], + "properties": { + "resultsPerPage": { "type": "integer", "minimum": 0 }, + "startIndex": { "type": "integer", "minimum": 0 }, + "totalResults": { "type": "integer", "minimum": 0 }, + "vulnerabilities": { + "type": "array", + "items": { + "type": "object", + "required": ["cve"], + "properties": { + "cve": { + "type": "object", + "required": ["id", "published", "lastModified", "descriptions"], + "properties": { + "id": { "type": "string" }, + "published": { "type": "string", "format": "date-time" }, + "lastModified": { "type": "string", "format": "date-time" }, + "vulnStatus": { "type": "string" }, + "sourceIdentifier": { "type": "string" }, + "descriptions": { + "type": "array", + "items": { + "type": "object", + "required": ["lang", "value"], + "properties": { + "lang": { "type": "string" }, + "value": { "type": "string" } + } + } + }, + "references": { + "type": "array", + "items": { + "type": "object", + "required": ["url"], + "properties": { + "url": { "type": "string", "format": "uri" }, + "source": { "type": "string" }, + "tags": { + "type": "array", + "items": { "type": "string" } + } + } + } + }, + "metrics": { + "type": "object", + "properties": { + "cvssMetricV2": { "$ref": "#/definitions/cvssMetricArray" }, + "cvssMetricV30": { "$ref": "#/definitions/cvssMetricArray" }, + "cvssMetricV31": { "$ref": "#/definitions/cvssMetricArray" } + } + }, + "configurations": { + "type": "object", + "properties": { + "nodes": { + "type": "array", + "items": { + "type": "object", + "properties": { + "cpeMatch": { + "type": "array", + "items": { + "type": "object", + "properties": { + "vulnerable": { "type": "boolean" }, + "criteria": { "type": "string" } + }, + "required": ["criteria"], + "additionalProperties": true + } + } + }, + "additionalProperties": true + } + } + }, + "additionalProperties": true + } + }, + "additionalProperties": true + } + }, + "additionalProperties": true + } + } + }, + "additionalProperties": true, + "definitions": { + "cvssMetricArray": { + "type": "array", + "items": { + "type": "object", + "properties": { + "cvssData": { + "type": "object", + "required": ["vectorString", "baseScore", "baseSeverity"], + "properties": { + "vectorString": { "type": "string" }, + "baseScore": { "type": "number" }, + "baseSeverity": { "type": "string" } + }, + "additionalProperties": true + } + }, + "additionalProperties": true + } + } + } +} diff --git a/src/StellaOps.Feedser.Source.Nvd/StellaOps.Feedser.Source.Nvd.csproj b/src/StellaOps.Feedser.Source.Nvd/StellaOps.Feedser.Source.Nvd.csproj new file mode 100644 index 00000000..74d98ea9 --- /dev/null +++ b/src/StellaOps.Feedser.Source.Nvd/StellaOps.Feedser.Source.Nvd.csproj @@ -0,0 +1,17 @@ + + + net10.0 + enable + enable + + + + + + + + + + + + diff --git a/src/StellaOps.Feedser.Source.Nvd/TASKS.md b/src/StellaOps.Feedser.Source.Nvd/TASKS.md new file mode 100644 index 00000000..47ea4434 --- /dev/null +++ b/src/StellaOps.Feedser.Source.Nvd/TASKS.md @@ -0,0 +1,15 @@ +# TASKS +| Task | Owner(s) | Depends on | Notes | +|---|---|---|---| +|Fetch job with sliding modified windows|BE-Conn-Nvd|Source.Common|**DONE** – windowed fetch implemented with overlap and raw doc persistence.| +|DTO schema + validation|BE-Conn-Nvd|Source.Common|**DONE** – schema validator enforced before DTO persistence.| +|Mapper to canonical model|BE-Conn-Nvd|Models|**DONE** – `NvdMapper` populates CVSS/CWE/CPE data.| +|Watermark repo usage|BE-Conn-Nvd|Storage.Mongo|**DONE** – cursor tracks windowStart/windowEnd and updates SourceState.| +|Integration test fixture isolation|QA|Storage.Mongo|**DONE** – connector tests reset Mongo/time fixtures between runs to avoid cross-test bleed.| +|Tests: golden pages + resume|QA|Tests|**DONE** – snapshot and resume coverage added across `NvdConnectorTests`.| +|Observability|BE-Conn-Nvd|Core|**DONE** – `NvdDiagnostics` meter tracks attempts/documents/failures with collector tests.| +|Change history snapshotting|BE-Conn-Nvd|Storage.Mongo|DONE – connector now records per-CVE snapshots with top-level diff metadata whenever canonical advisories change.| +|Pagination for windows over page limit|BE-Conn-Nvd|Source.Common|**DONE** – additional page fetcher honors `startIndex`; covered by multipage tests.| +|Schema validation quarantine path|BE-Conn-Nvd|Storage.Mongo|**DONE** – schema failures mark documents failed and metrics assert quarantine.| +|FEEDCONN-NVD-02-004 NVD CVSS & CWE precedence payloads|BE-Conn-Nvd|Models `FEEDMODELS-SCHEMA-01-002`|**DONE (2025-10-11)** – CVSS metrics now carry provenance masks, CWE weaknesses emit normalized references, and fixtures cover the additional precedence data.| +|FEEDCONN-NVD-02-005 NVD merge/export parity regression|BE-Conn-Nvd, BE-Merge|Merge `FEEDMERGE-ENGINE-04-003`|TODO – Add integration fixtures validating credit/reference union post-merge and ensure exporters emit the enriched metadata consistently.| diff --git a/src/StellaOps.Feedser.Source.Osv.Tests/Fixtures/osv-ghsa.ghsa.json b/src/StellaOps.Feedser.Source.Osv.Tests/Fixtures/osv-ghsa.ghsa.json new file mode 100644 index 00000000..718d4585 --- /dev/null +++ b/src/StellaOps.Feedser.Source.Osv.Tests/Fixtures/osv-ghsa.ghsa.json @@ -0,0 +1,1183 @@ +[ + { + "advisoryKey": "GHSA-77vh-xpmg-72qh", + "affectedPackages": [ + { + "type": "semver", + "identifier": "pkg:golang/github.com/opencontainers/image-spec", + "platform": "go", + "versionRanges": [ + { + "fixedVersion": "1.0.2", + "introducedVersion": null, + "lastAffectedVersion": null, + "primitives": null, + "provenance": { + "source": "ghsa", + "kind": "range", + "value": "pkg:golang/github.com/opencontainers/image-spec", + "decisionReason": null, + "recordedAt": "2023-01-09T05:05:32+00:00", + "fieldMask": [ + "affectedpackages[].versionranges[]" + ] + }, + "rangeExpression": "< 1.0.2", + "rangeKind": "semver" + } + ], + "normalizedVersions": [], + "statuses": [], + "provenance": [ + { + "source": "ghsa", + "kind": "package", + "value": "pkg:golang/github.com/opencontainers/image-spec", + "decisionReason": null, + "recordedAt": "2023-01-09T05:05:32+00:00", + "fieldMask": [ + "affectedpackages[]" + ] + } + ] + } + ], + "aliases": [ + "GHSA-77vh-xpmg-72qh" + ], + "cvssMetrics": [ + { + "baseScore": 3, + "baseSeverity": "low", + "provenance": { + "source": "ghsa", + "kind": "cvss", + "value": "CVSS:3.1/AV:N/AC:H/PR:L/UI:R/S:C/C:N/I:L/A:N", + "decisionReason": null, + "recordedAt": "2023-01-09T05:05:32+00:00", + "fieldMask": [ + "cvssmetrics[]" + ] + }, + "vector": "CVSS:3.1/AV:N/AC:H/PR:L/UI:R/S:C/C:N/I:L/A:N", + "version": "3.1" + } + ], + "exploitKnown": false, + "language": "en", + "modified": "2023-01-09T05:05:32+00:00", + "provenance": [ + { + "source": "ghsa", + "kind": "map", + "value": "GHSA-77vh-xpmg-72qh", + "decisionReason": null, + "recordedAt": "2023-01-09T05:05:32+00:00", + "fieldMask": [ + "advisory" + ] + } + ], + "published": "2021-11-18T16:02:41+00:00", + "references": [ + { + "kind": "advisory", + "provenance": { + "source": "ghsa", + "kind": "reference", + "value": "https://github.com/advisories/GHSA-77vh-xpmg-72qh", + "decisionReason": null, + "recordedAt": "2023-01-09T05:05:32+00:00", + "fieldMask": [ + "references[]" + ] + }, + "sourceTag": "github.com", + "summary": null, + "url": "https://github.com/advisories/GHSA-77vh-xpmg-72qh" + }, + { + "kind": "advisory", + "provenance": { + "source": "ghsa", + "kind": "reference", + "value": "https://github.com/opencontainers/distribution-spec/security/advisories/GHSA-mc8v-mgrf-8f4m", + "decisionReason": null, + "recordedAt": "2023-01-09T05:05:32+00:00", + "fieldMask": [ + "references[]" + ] + }, + "sourceTag": "github.com", + "summary": null, + "url": "https://github.com/opencontainers/distribution-spec/security/advisories/GHSA-mc8v-mgrf-8f4m" + }, + { + "kind": "patch", + "provenance": { + "source": "ghsa", + "kind": "reference", + "value": "https://github.com/opencontainers/image-spec/commit/693428a734f5bab1a84bd2f990d92ef1111cd60c", + "decisionReason": null, + "recordedAt": "2023-01-09T05:05:32+00:00", + "fieldMask": [ + "references[]" + ] + }, + "sourceTag": "github.com", + "summary": null, + "url": "https://github.com/opencontainers/image-spec/commit/693428a734f5bab1a84bd2f990d92ef1111cd60c" + }, + { + "kind": "patch", + "provenance": { + "source": "ghsa", + "kind": "reference", + "value": "https://github.com/opencontainers/image-spec/releases/tag/v1.0.2", + "decisionReason": null, + "recordedAt": "2023-01-09T05:05:32+00:00", + "fieldMask": [ + "references[]" + ] + }, + "sourceTag": "github.com", + "summary": null, + "url": "https://github.com/opencontainers/image-spec/releases/tag/v1.0.2" + }, + { + "kind": "advisory", + "provenance": { + "source": "ghsa", + "kind": "reference", + "value": "https://github.com/opencontainers/image-spec/security/advisories/GHSA-77vh-xpmg-72qh", + "decisionReason": null, + "recordedAt": "2023-01-09T05:05:32+00:00", + "fieldMask": [ + "references[]" + ] + }, + "sourceTag": "github.com", + "summary": null, + "url": "https://github.com/opencontainers/image-spec/security/advisories/GHSA-77vh-xpmg-72qh" + } + ], + "severity": "low", + "summary": "### Impact\nIn the OCI Image Specification version 1.0.1 and prior, manifest and index documents are not self-describing and documents with a single digest could be interpreted as either a manifest or an index.\n\n### Patches\nThe Image Specification will be updated to recommend that both manifest and index documents contain a `mediaType` field to identify the type of document.\nRelease [v1.0.2](https://github.com/opencontainers/image-spec/releases/tag/v1.0.2) includes these updates.\n\n### Workarounds\nSoftware attempting to deserialize an ambiguous document may reject the document if it contains both “manifests” and “layers” fields or “manifests” and “config” fields.\n\n### References\nhttps://github.com/opencontainers/distribution-spec/security/advisories/GHSA-mc8v-mgrf-8f4m\n\n### For more information\nIf you have any questions or comments about this advisory:\n* Open an issue in https://github.com/opencontainers/image-spec\n* Email us at [security@opencontainers.org](mailto:security@opencontainers.org)\n* https://github.com/opencontainers/image-spec/commits/v1.0.2", + "title": "Clarify `mediaType` handling" + }, + { + "advisoryKey": "GHSA-7rjr-3q55-vv33", + "affectedPackages": [ + { + "type": "semver", + "identifier": "pkg:maven/org.apache.logging.log4j/log4j-core", + "platform": "maven", + "versionRanges": [ + { + "fixedVersion": "2.16.0", + "introducedVersion": "2.13.0", + "lastAffectedVersion": null, + "primitives": null, + "provenance": { + "source": "ghsa", + "kind": "range", + "value": "pkg:maven/org.apache.logging.log4j/log4j-core", + "decisionReason": null, + "recordedAt": "2025-05-09T12:28:41+00:00", + "fieldMask": [ + "affectedpackages[].versionranges[]" + ] + }, + "rangeExpression": ">= 2.13.0, < 2.16.0", + "rangeKind": "semver" + } + ], + "normalizedVersions": [], + "statuses": [], + "provenance": [ + { + "source": "ghsa", + "kind": "package", + "value": "pkg:maven/org.apache.logging.log4j/log4j-core", + "decisionReason": null, + "recordedAt": "2025-05-09T12:28:41+00:00", + "fieldMask": [ + "affectedpackages[]" + ] + } + ] + }, + { + "type": "semver", + "identifier": "pkg:maven/org.apache.logging.log4j/log4j-core", + "platform": "maven", + "versionRanges": [ + { + "fixedVersion": "2.12.2", + "introducedVersion": null, + "lastAffectedVersion": null, + "primitives": null, + "provenance": { + "source": "ghsa", + "kind": "range", + "value": "pkg:maven/org.apache.logging.log4j/log4j-core", + "decisionReason": null, + "recordedAt": "2025-05-09T12:28:41+00:00", + "fieldMask": [ + "affectedpackages[].versionranges[]" + ] + }, + "rangeExpression": "< 2.12.2", + "rangeKind": "semver" + } + ], + "normalizedVersions": [], + "statuses": [], + "provenance": [ + { + "source": "ghsa", + "kind": "package", + "value": "pkg:maven/org.apache.logging.log4j/log4j-core", + "decisionReason": null, + "recordedAt": "2025-05-09T12:28:41+00:00", + "fieldMask": [ + "affectedpackages[]" + ] + } + ] + }, + { + "type": "semver", + "identifier": "pkg:maven/org.ops4j.pax.logging/pax-logging-log4j2", + "platform": "maven", + "versionRanges": [ + { + "fixedVersion": "1.9.2", + "introducedVersion": "1.8.0", + "lastAffectedVersion": null, + "primitives": null, + "provenance": { + "source": "ghsa", + "kind": "range", + "value": "pkg:maven/org.ops4j.pax.logging/pax-logging-log4j2", + "decisionReason": null, + "recordedAt": "2025-05-09T12:28:41+00:00", + "fieldMask": [ + "affectedpackages[].versionranges[]" + ] + }, + "rangeExpression": ">= 1.8.0, < 1.9.2", + "rangeKind": "semver" + } + ], + "normalizedVersions": [], + "statuses": [], + "provenance": [ + { + "source": "ghsa", + "kind": "package", + "value": "pkg:maven/org.ops4j.pax.logging/pax-logging-log4j2", + "decisionReason": null, + "recordedAt": "2025-05-09T12:28:41+00:00", + "fieldMask": [ + "affectedpackages[]" + ] + } + ] + }, + { + "type": "semver", + "identifier": "pkg:maven/org.ops4j.pax.logging/pax-logging-log4j2", + "platform": "maven", + "versionRanges": [ + { + "fixedVersion": "1.10.8", + "introducedVersion": "1.10.0", + "lastAffectedVersion": null, + "primitives": null, + "provenance": { + "source": "ghsa", + "kind": "range", + "value": "pkg:maven/org.ops4j.pax.logging/pax-logging-log4j2", + "decisionReason": null, + "recordedAt": "2025-05-09T12:28:41+00:00", + "fieldMask": [ + "affectedpackages[].versionranges[]" + ] + }, + "rangeExpression": ">= 1.10.0, < 1.10.8", + "rangeKind": "semver" + } + ], + "normalizedVersions": [], + "statuses": [], + "provenance": [ + { + "source": "ghsa", + "kind": "package", + "value": "pkg:maven/org.ops4j.pax.logging/pax-logging-log4j2", + "decisionReason": null, + "recordedAt": "2025-05-09T12:28:41+00:00", + "fieldMask": [ + "affectedpackages[]" + ] + } + ] + }, + { + "type": "semver", + "identifier": "pkg:maven/org.ops4j.pax.logging/pax-logging-log4j2", + "platform": "maven", + "versionRanges": [ + { + "fixedVersion": "1.11.11", + "introducedVersion": "1.11.0", + "lastAffectedVersion": null, + "primitives": null, + "provenance": { + "source": "ghsa", + "kind": "range", + "value": "pkg:maven/org.ops4j.pax.logging/pax-logging-log4j2", + "decisionReason": null, + "recordedAt": "2025-05-09T12:28:41+00:00", + "fieldMask": [ + "affectedpackages[].versionranges[]" + ] + }, + "rangeExpression": ">= 1.11.0, < 1.11.11", + "rangeKind": "semver" + } + ], + "normalizedVersions": [], + "statuses": [], + "provenance": [ + { + "source": "ghsa", + "kind": "package", + "value": "pkg:maven/org.ops4j.pax.logging/pax-logging-log4j2", + "decisionReason": null, + "recordedAt": "2025-05-09T12:28:41+00:00", + "fieldMask": [ + "affectedpackages[]" + ] + } + ] + }, + { + "type": "semver", + "identifier": "pkg:maven/org.ops4j.pax.logging/pax-logging-log4j2", + "platform": "maven", + "versionRanges": [ + { + "fixedVersion": "2.0.12", + "introducedVersion": "2.0.0", + "lastAffectedVersion": null, + "primitives": null, + "provenance": { + "source": "ghsa", + "kind": "range", + "value": "pkg:maven/org.ops4j.pax.logging/pax-logging-log4j2", + "decisionReason": null, + "recordedAt": "2025-05-09T12:28:41+00:00", + "fieldMask": [ + "affectedpackages[].versionranges[]" + ] + }, + "rangeExpression": ">= 2.0.0, < 2.0.12", + "rangeKind": "semver" + } + ], + "normalizedVersions": [], + "statuses": [], + "provenance": [ + { + "source": "ghsa", + "kind": "package", + "value": "pkg:maven/org.ops4j.pax.logging/pax-logging-log4j2", + "decisionReason": null, + "recordedAt": "2025-05-09T12:28:41+00:00", + "fieldMask": [ + "affectedpackages[]" + ] + } + ] + } + ], + "aliases": [ + "CVE-2021-45046", + "GHSA-7rjr-3q55-vv33" + ], + "cvssMetrics": [ + { + "baseScore": 9.1, + "baseSeverity": "critical", + "provenance": { + "source": "ghsa", + "kind": "cvss", + "value": "CVSS:3.1/AV:N/AC:H/PR:N/UI:N/S:C/C:H/I:H/A:H", + "decisionReason": null, + "recordedAt": "2025-05-09T12:28:41+00:00", + "fieldMask": [ + "cvssmetrics[]" + ] + }, + "vector": "CVSS:3.1/AV:N/AC:H/PR:N/UI:N/S:C/C:H/I:H/A:H", + "version": "3.1" + } + ], + "exploitKnown": false, + "language": "en", + "modified": "2025-05-09T12:28:41+00:00", + "provenance": [ + { + "source": "ghsa", + "kind": "map", + "value": "GHSA-7rjr-3q55-vv33", + "decisionReason": null, + "recordedAt": "2025-05-09T12:28:41+00:00", + "fieldMask": [ + "advisory" + ] + } + ], + "published": "2021-12-14T18:01:28+00:00", + "references": [ + { + "kind": "advisory", + "provenance": { + "source": "ghsa", + "kind": "reference", + "value": "http://www.openwall.com/lists/oss-security/2021/12/14/4", + "decisionReason": null, + "recordedAt": "2025-05-09T12:28:41+00:00", + "fieldMask": [ + "references[]" + ] + }, + "sourceTag": "www.openwall.com", + "summary": null, + "url": "http://www.openwall.com/lists/oss-security/2021/12/14/4" + }, + { + "kind": "advisory", + "provenance": { + "source": "ghsa", + "kind": "reference", + "value": "http://www.openwall.com/lists/oss-security/2021/12/15/3", + "decisionReason": null, + "recordedAt": "2025-05-09T12:28:41+00:00", + "fieldMask": [ + "references[]" + ] + }, + "sourceTag": "www.openwall.com", + "summary": null, + "url": "http://www.openwall.com/lists/oss-security/2021/12/15/3" + }, + { + "kind": "advisory", + "provenance": { + "source": "ghsa", + "kind": "reference", + "value": "http://www.openwall.com/lists/oss-security/2021/12/18/1", + "decisionReason": null, + "recordedAt": "2025-05-09T12:28:41+00:00", + "fieldMask": [ + "references[]" + ] + }, + "sourceTag": "www.openwall.com", + "summary": null, + "url": "http://www.openwall.com/lists/oss-security/2021/12/18/1" + }, + { + "kind": null, + "provenance": { + "source": "ghsa", + "kind": "reference", + "value": "https://cert-portal.siemens.com/productcert/pdf/ssa-397453.pdf", + "decisionReason": null, + "recordedAt": "2025-05-09T12:28:41+00:00", + "fieldMask": [ + "references[]" + ] + }, + "sourceTag": "cert-portal.siemens.com", + "summary": null, + "url": "https://cert-portal.siemens.com/productcert/pdf/ssa-397453.pdf" + }, + { + "kind": null, + "provenance": { + "source": "ghsa", + "kind": "reference", + "value": "https://cert-portal.siemens.com/productcert/pdf/ssa-479842.pdf", + "decisionReason": null, + "recordedAt": "2025-05-09T12:28:41+00:00", + "fieldMask": [ + "references[]" + ] + }, + "sourceTag": "cert-portal.siemens.com", + "summary": null, + "url": "https://cert-portal.siemens.com/productcert/pdf/ssa-479842.pdf" + }, + { + "kind": null, + "provenance": { + "source": "ghsa", + "kind": "reference", + "value": "https://cert-portal.siemens.com/productcert/pdf/ssa-661247.pdf", + "decisionReason": null, + "recordedAt": "2025-05-09T12:28:41+00:00", + "fieldMask": [ + "references[]" + ] + }, + "sourceTag": "cert-portal.siemens.com", + "summary": null, + "url": "https://cert-portal.siemens.com/productcert/pdf/ssa-661247.pdf" + }, + { + "kind": null, + "provenance": { + "source": "ghsa", + "kind": "reference", + "value": "https://cert-portal.siemens.com/productcert/pdf/ssa-714170.pdf", + "decisionReason": null, + "recordedAt": "2025-05-09T12:28:41+00:00", + "fieldMask": [ + "references[]" + ] + }, + "sourceTag": "cert-portal.siemens.com", + "summary": null, + "url": "https://cert-portal.siemens.com/productcert/pdf/ssa-714170.pdf" + }, + { + "kind": "advisory", + "provenance": { + "source": "ghsa", + "kind": "reference", + "value": "https://github.com/advisories/GHSA-7rjr-3q55-vv33", + "decisionReason": null, + "recordedAt": "2025-05-09T12:28:41+00:00", + "fieldMask": [ + "references[]" + ] + }, + "sourceTag": "github.com", + "summary": null, + "url": "https://github.com/advisories/GHSA-7rjr-3q55-vv33" + }, + { + "kind": "advisory", + "provenance": { + "source": "ghsa", + "kind": "reference", + "value": "https://github.com/advisories/GHSA-jfh8-c2jp-5v3q", + "decisionReason": null, + "recordedAt": "2025-05-09T12:28:41+00:00", + "fieldMask": [ + "references[]" + ] + }, + "sourceTag": "github.com", + "summary": null, + "url": "https://github.com/advisories/GHSA-jfh8-c2jp-5v3q" + }, + { + "kind": null, + "provenance": { + "source": "ghsa", + "kind": "reference", + "value": "https://lists.fedoraproject.org/archives/list/package-announce@lists.fedoraproject.org/message/EOKPQGV24RRBBI4TBZUDQMM4MEH7MXCY", + "decisionReason": null, + "recordedAt": "2025-05-09T12:28:41+00:00", + "fieldMask": [ + "references[]" + ] + }, + "sourceTag": "lists.fedoraproject.org", + "summary": null, + "url": "https://lists.fedoraproject.org/archives/list/package-announce@lists.fedoraproject.org/message/EOKPQGV24RRBBI4TBZUDQMM4MEH7MXCY" + }, + { + "kind": null, + "provenance": { + "source": "ghsa", + "kind": "reference", + "value": "https://lists.fedoraproject.org/archives/list/package-announce@lists.fedoraproject.org/message/SIG7FZULMNK2XF6FZRU4VWYDQXNMUGAJ", + "decisionReason": null, + "recordedAt": "2025-05-09T12:28:41+00:00", + "fieldMask": [ + "references[]" + ] + }, + "sourceTag": "lists.fedoraproject.org", + "summary": null, + "url": "https://lists.fedoraproject.org/archives/list/package-announce@lists.fedoraproject.org/message/SIG7FZULMNK2XF6FZRU4VWYDQXNMUGAJ" + }, + { + "kind": "advisory", + "provenance": { + "source": "ghsa", + "kind": "reference", + "value": "https://logging.apache.org/log4j/2.x/security.html", + "decisionReason": null, + "recordedAt": "2025-05-09T12:28:41+00:00", + "fieldMask": [ + "references[]" + ] + }, + "sourceTag": "logging.apache.org", + "summary": null, + "url": "https://logging.apache.org/log4j/2.x/security.html" + }, + { + "kind": "advisory", + "provenance": { + "source": "ghsa", + "kind": "reference", + "value": "https://nvd.nist.gov/vuln/detail/CVE-2021-45046", + "decisionReason": null, + "recordedAt": "2025-05-09T12:28:41+00:00", + "fieldMask": [ + "references[]" + ] + }, + "sourceTag": "nvd.nist.gov", + "summary": null, + "url": "https://nvd.nist.gov/vuln/detail/CVE-2021-45046" + }, + { + "kind": null, + "provenance": { + "source": "ghsa", + "kind": "reference", + "value": "https://psirt.global.sonicwall.com/vuln-detail/SNWLID-2021-0032", + "decisionReason": null, + "recordedAt": "2025-05-09T12:28:41+00:00", + "fieldMask": [ + "references[]" + ] + }, + "sourceTag": "psirt.global.sonicwall.com", + "summary": null, + "url": "https://psirt.global.sonicwall.com/vuln-detail/SNWLID-2021-0032" + }, + { + "kind": "advisory", + "provenance": { + "source": "ghsa", + "kind": "reference", + "value": "https://sec.cloudapps.cisco.com/security/center/content/CiscoSecurityAdvisory/cisco-sa-apache-log4j-qRuKNEbd", + "decisionReason": null, + "recordedAt": "2025-05-09T12:28:41+00:00", + "fieldMask": [ + "references[]" + ] + }, + "sourceTag": "sec.cloudapps.cisco.com", + "summary": null, + "url": "https://sec.cloudapps.cisco.com/security/center/content/CiscoSecurityAdvisory/cisco-sa-apache-log4j-qRuKNEbd" + }, + { + "kind": "advisory", + "provenance": { + "source": "ghsa", + "kind": "reference", + "value": "https://security.gentoo.org/glsa/202310-16", + "decisionReason": null, + "recordedAt": "2025-05-09T12:28:41+00:00", + "fieldMask": [ + "references[]" + ] + }, + "sourceTag": "security.gentoo.org", + "summary": null, + "url": "https://security.gentoo.org/glsa/202310-16" + }, + { + "kind": "advisory", + "provenance": { + "source": "ghsa", + "kind": "reference", + "value": "https://www.cve.org/CVERecord?id=CVE-2021-44228", + "decisionReason": null, + "recordedAt": "2025-05-09T12:28:41+00:00", + "fieldMask": [ + "references[]" + ] + }, + "sourceTag": "www.cve.org", + "summary": null, + "url": "https://www.cve.org/CVERecord?id=CVE-2021-44228" + }, + { + "kind": "advisory", + "provenance": { + "source": "ghsa", + "kind": "reference", + "value": "https://www.debian.org/security/2021/dsa-5022", + "decisionReason": null, + "recordedAt": "2025-05-09T12:28:41+00:00", + "fieldMask": [ + "references[]" + ] + }, + "sourceTag": "www.debian.org", + "summary": null, + "url": "https://www.debian.org/security/2021/dsa-5022" + }, + { + "kind": "advisory", + "provenance": { + "source": "ghsa", + "kind": "reference", + "value": "https://www.intel.com/content/www/us/en/security-center/advisory/intel-sa-00646.html", + "decisionReason": null, + "recordedAt": "2025-05-09T12:28:41+00:00", + "fieldMask": [ + "references[]" + ] + }, + "sourceTag": "www.intel.com", + "summary": null, + "url": "https://www.intel.com/content/www/us/en/security-center/advisory/intel-sa-00646.html" + }, + { + "kind": null, + "provenance": { + "source": "ghsa", + "kind": "reference", + "value": "https://www.kb.cert.org/vuls/id/930724", + "decisionReason": null, + "recordedAt": "2025-05-09T12:28:41+00:00", + "fieldMask": [ + "references[]" + ] + }, + "sourceTag": "www.kb.cert.org", + "summary": null, + "url": "https://www.kb.cert.org/vuls/id/930724" + }, + { + "kind": "advisory", + "provenance": { + "source": "ghsa", + "kind": "reference", + "value": "https://www.openwall.com/lists/oss-security/2021/12/14/4", + "decisionReason": null, + "recordedAt": "2025-05-09T12:28:41+00:00", + "fieldMask": [ + "references[]" + ] + }, + "sourceTag": "www.openwall.com", + "summary": null, + "url": "https://www.openwall.com/lists/oss-security/2021/12/14/4" + }, + { + "kind": "advisory", + "provenance": { + "source": "ghsa", + "kind": "reference", + "value": "https://www.oracle.com/security-alerts/alert-cve-2021-44228.html", + "decisionReason": null, + "recordedAt": "2025-05-09T12:28:41+00:00", + "fieldMask": [ + "references[]" + ] + }, + "sourceTag": "www.oracle.com", + "summary": null, + "url": "https://www.oracle.com/security-alerts/alert-cve-2021-44228.html" + }, + { + "kind": "advisory", + "provenance": { + "source": "ghsa", + "kind": "reference", + "value": "https://www.oracle.com/security-alerts/cpuapr2022.html", + "decisionReason": null, + "recordedAt": "2025-05-09T12:28:41+00:00", + "fieldMask": [ + "references[]" + ] + }, + "sourceTag": "www.oracle.com", + "summary": null, + "url": "https://www.oracle.com/security-alerts/cpuapr2022.html" + }, + { + "kind": "advisory", + "provenance": { + "source": "ghsa", + "kind": "reference", + "value": "https://www.oracle.com/security-alerts/cpujan2022.html", + "decisionReason": null, + "recordedAt": "2025-05-09T12:28:41+00:00", + "fieldMask": [ + "references[]" + ] + }, + "sourceTag": "www.oracle.com", + "summary": null, + "url": "https://www.oracle.com/security-alerts/cpujan2022.html" + }, + { + "kind": "advisory", + "provenance": { + "source": "ghsa", + "kind": "reference", + "value": "https://www.oracle.com/security-alerts/cpujul2022.html", + "decisionReason": null, + "recordedAt": "2025-05-09T12:28:41+00:00", + "fieldMask": [ + "references[]" + ] + }, + "sourceTag": "www.oracle.com", + "summary": null, + "url": "https://www.oracle.com/security-alerts/cpujul2022.html" + } + ], + "severity": "critical", + "summary": "# Impact\n\nThe fix to address [CVE-2021-44228](https://nvd.nist.gov/vuln/detail/CVE-2021-44228) in Apache Log4j 2.15.0 was incomplete in certain non-default configurations. This could allow attackers with control over Thread Context Map (MDC) input data when the logging configuration uses a non-default Pattern Layout with either a Context Lookup (for example, $${ctx:loginId}) or a Thread Context Map pattern (%X, %mdc, or %MDC) to craft malicious input data using a JNDI Lookup pattern resulting in a remote code execution (RCE) attack. \n\n## Affected packages\nOnly the `org.apache.logging.log4j:log4j-core` package is directly affected by this vulnerability. The `org.apache.logging.log4j:log4j-api` should be kept at the same version as the `org.apache.logging.log4j:log4j-core` package to ensure compatability if in use.\n\n# Mitigation\n\nLog4j 2.16.0 fixes this issue by removing support for message lookup patterns and disabling JNDI functionality by default. This issue can be mitigated in prior releases (< 2.16.0) by removing the JndiLookup class from the classpath (example: zip -q -d log4j-core-*.jar org/apache/logging/log4j/core/lookup/JndiLookup.class).\n\nLog4j 2.15.0 restricts JNDI LDAP lookups to localhost by default. Note that previous mitigations involving configuration such as to set the system property `log4j2.formatMsgNoLookups` to `true` do NOT mitigate this specific vulnerability.", + "title": "Incomplete fix for Apache Log4j vulnerability" + }, + { + "advisoryKey": "GHSA-cjjf-27cc-pvmv", + "affectedPackages": [ + { + "type": "semver", + "identifier": "pyload-ng", + "platform": "pip", + "versionRanges": [ + { + "fixedVersion": "0.5.0b3.dev91", + "introducedVersion": null, + "lastAffectedVersion": null, + "primitives": null, + "provenance": { + "source": "ghsa", + "kind": "range", + "value": "pyload-ng", + "decisionReason": null, + "recordedAt": "2025-10-09T15:19:48+00:00", + "fieldMask": [ + "affectedpackages[].versionranges[]" + ] + }, + "rangeExpression": "< 0.5.0b3.dev91", + "rangeKind": "semver" + } + ], + "normalizedVersions": [], + "statuses": [], + "provenance": [ + { + "source": "ghsa", + "kind": "package", + "value": "pyload-ng", + "decisionReason": null, + "recordedAt": "2025-10-09T15:19:48+00:00", + "fieldMask": [ + "affectedpackages[]" + ] + } + ] + } + ], + "aliases": [ + "CVE-2025-61773", + "GHSA-cjjf-27cc-pvmv" + ], + "cvssMetrics": [ + { + "baseScore": 8.1, + "baseSeverity": "high", + "provenance": { + "source": "ghsa", + "kind": "cvss", + "value": "CVSS:3.1/AV:N/AC:L/PR:N/UI:R/S:U/C:H/I:H/A:N", + "decisionReason": null, + "recordedAt": "2025-10-09T15:19:48+00:00", + "fieldMask": [ + "cvssmetrics[]" + ] + }, + "vector": "CVSS:3.1/AV:N/AC:L/PR:N/UI:R/S:U/C:H/I:H/A:N", + "version": "3.1" + } + ], + "exploitKnown": false, + "language": "en", + "modified": "2025-10-09T15:19:48+00:00", + "provenance": [ + { + "source": "ghsa", + "kind": "map", + "value": "GHSA-cjjf-27cc-pvmv", + "decisionReason": null, + "recordedAt": "2025-10-09T15:19:48+00:00", + "fieldMask": [ + "advisory" + ] + } + ], + "published": "2025-10-09T15:19:48+00:00", + "references": [ + { + "kind": "advisory", + "provenance": { + "source": "ghsa", + "kind": "reference", + "value": "https://github.com/advisories/GHSA-cjjf-27cc-pvmv", + "decisionReason": null, + "recordedAt": "2025-10-09T15:19:48+00:00", + "fieldMask": [ + "references[]" + ] + }, + "sourceTag": "github.com", + "summary": null, + "url": "https://github.com/advisories/GHSA-cjjf-27cc-pvmv" + }, + { + "kind": "patch", + "provenance": { + "source": "ghsa", + "kind": "reference", + "value": "https://github.com/pyload/pyload/commit/5823327d0b797161c7195a1f660266d30a69f0ca", + "decisionReason": null, + "recordedAt": "2025-10-09T15:19:48+00:00", + "fieldMask": [ + "references[]" + ] + }, + "sourceTag": "github.com", + "summary": null, + "url": "https://github.com/pyload/pyload/commit/5823327d0b797161c7195a1f660266d30a69f0ca" + }, + { + "kind": "patch", + "provenance": { + "source": "ghsa", + "kind": "reference", + "value": "https://github.com/pyload/pyload/pull/4624", + "decisionReason": null, + "recordedAt": "2025-10-09T15:19:48+00:00", + "fieldMask": [ + "references[]" + ] + }, + "sourceTag": "github.com", + "summary": null, + "url": "https://github.com/pyload/pyload/pull/4624" + }, + { + "kind": "advisory", + "provenance": { + "source": "ghsa", + "kind": "reference", + "value": "https://github.com/pyload/pyload/security/advisories/GHSA-cjjf-27cc-pvmv", + "decisionReason": null, + "recordedAt": "2025-10-09T15:19:48+00:00", + "fieldMask": [ + "references[]" + ] + }, + "sourceTag": "github.com", + "summary": null, + "url": "https://github.com/pyload/pyload/security/advisories/GHSA-cjjf-27cc-pvmv" + } + ], + "severity": "high", + "summary": "### Summary\npyLoad web interface contained insufficient input validation in both the Captcha script endpoint and the Click'N'Load (CNL) Blueprint. This flaw allowed untrusted user input to be processed unsafely, which could be exploited by an attacker to inject arbitrary content into the web UI or manipulate request handling. The vulnerability could lead to client-side code execution (XSS) or other unintended behaviors when a malicious payload is submitted.\n\nuser-supplied parameters from HTTP requests were not adequately validated or sanitized before being passed into the application logic and response generation. This allowed crafted input to alter the expected execution flow.\n CNL (Click'N'Load) blueprint exposed unsafe handling of untrusted parameters in HTTP requests. The application did not consistently enforce input validation or encoding, making it possible for an attacker to craft malicious requests.\n\n### PoC\n\n1. Run a vulnerable version of pyLoad prior to commit [`f9d27f2`](https://github.com/pyload/pyload/pull/4624).\n2. Start the web UI and access the Captcha or CNL endpoints.\n3. Submit a crafted request containing malicious JavaScript payloads in unvalidated parameters (`/flash/addcrypted2?jk=function(){alert(1)}&crypted=12345`).\n4. Observe that the payload is reflected and executed in the client’s browser, demonstrating cross-site scripting (XSS).\n\nExample request:\n\n```http\nGET /flash/addcrypted2?jk=function(){alert(1)}&crypted=12345 HTTP/1.1\nHost: 127.0.0.1:8000\nContent-Type: application/x-www-form-urlencoded\nContent-Length: 107\n```\n\n### Impact\n\nExploiting this vulnerability allows an attacker to inject and execute arbitrary JavaScript within the browser session of a user accessing the pyLoad Web UI. In practice, this means an attacker could impersonate an administrator, steal authentication cookies or tokens, and perform unauthorized actions on behalf of the victim. Because the affected endpoints are part of the core interface, a successful attack undermines the trust and security of the entire application, potentially leading to a full compromise of the management interface and the data it controls. The impact is particularly severe in cases where the Web UI is exposed over a network without additional access restrictions, as it enables remote attackers to directly target users with crafted links or requests that trigger the vulnerability.", + "title": "pyLoad CNL and captcha handlers allow Code Injection via unsanitized parameters" + }, + { + "advisoryKey": "GHSA-wv4w-6qv2-qqfg", + "affectedPackages": [ + { + "type": "semver", + "identifier": "social-auth-app-django", + "platform": "pip", + "versionRanges": [ + { + "fixedVersion": "5.6.0", + "introducedVersion": null, + "lastAffectedVersion": null, + "primitives": null, + "provenance": { + "source": "ghsa", + "kind": "range", + "value": "social-auth-app-django", + "decisionReason": null, + "recordedAt": "2025-10-09T17:08:06+00:00", + "fieldMask": [ + "affectedpackages[].versionranges[]" + ] + }, + "rangeExpression": "< 5.6.0", + "rangeKind": "semver" + } + ], + "normalizedVersions": [], + "statuses": [], + "provenance": [ + { + "source": "ghsa", + "kind": "package", + "value": "social-auth-app-django", + "decisionReason": null, + "recordedAt": "2025-10-09T17:08:06+00:00", + "fieldMask": [ + "affectedpackages[]" + ] + } + ] + } + ], + "aliases": [ + "CVE-2025-61783", + "GHSA-wv4w-6qv2-qqfg" + ], + "cvssMetrics": [], + "exploitKnown": false, + "language": "en", + "modified": "2025-10-09T17:08:06+00:00", + "provenance": [ + { + "source": "ghsa", + "kind": "map", + "value": "GHSA-wv4w-6qv2-qqfg", + "decisionReason": null, + "recordedAt": "2025-10-09T17:08:06+00:00", + "fieldMask": [ + "advisory" + ] + } + ], + "published": "2025-10-09T17:08:05+00:00", + "references": [ + { + "kind": "advisory", + "provenance": { + "source": "ghsa", + "kind": "reference", + "value": "https://github.com/advisories/GHSA-wv4w-6qv2-qqfg", + "decisionReason": null, + "recordedAt": "2025-10-09T17:08:06+00:00", + "fieldMask": [ + "references[]" + ] + }, + "sourceTag": "github.com", + "summary": null, + "url": "https://github.com/advisories/GHSA-wv4w-6qv2-qqfg" + }, + { + "kind": "patch", + "provenance": { + "source": "ghsa", + "kind": "reference", + "value": "https://github.com/python-social-auth/social-app-django/commit/10c80e2ebabeccd4e9c84ad0e16e1db74148ed4c", + "decisionReason": null, + "recordedAt": "2025-10-09T17:08:06+00:00", + "fieldMask": [ + "references[]" + ] + }, + "sourceTag": "github.com", + "summary": null, + "url": "https://github.com/python-social-auth/social-app-django/commit/10c80e2ebabeccd4e9c84ad0e16e1db74148ed4c" + }, + { + "kind": null, + "provenance": { + "source": "ghsa", + "kind": "reference", + "value": "https://github.com/python-social-auth/social-app-django/issues/220", + "decisionReason": null, + "recordedAt": "2025-10-09T17:08:06+00:00", + "fieldMask": [ + "references[]" + ] + }, + "sourceTag": "github.com", + "summary": null, + "url": "https://github.com/python-social-auth/social-app-django/issues/220" + }, + { + "kind": null, + "provenance": { + "source": "ghsa", + "kind": "reference", + "value": "https://github.com/python-social-auth/social-app-django/issues/231", + "decisionReason": null, + "recordedAt": "2025-10-09T17:08:06+00:00", + "fieldMask": [ + "references[]" + ] + }, + "sourceTag": "github.com", + "summary": null, + "url": "https://github.com/python-social-auth/social-app-django/issues/231" + }, + { + "kind": null, + "provenance": { + "source": "ghsa", + "kind": "reference", + "value": "https://github.com/python-social-auth/social-app-django/issues/634", + "decisionReason": null, + "recordedAt": "2025-10-09T17:08:06+00:00", + "fieldMask": [ + "references[]" + ] + }, + "sourceTag": "github.com", + "summary": null, + "url": "https://github.com/python-social-auth/social-app-django/issues/634" + }, + { + "kind": "patch", + "provenance": { + "source": "ghsa", + "kind": "reference", + "value": "https://github.com/python-social-auth/social-app-django/pull/803", + "decisionReason": null, + "recordedAt": "2025-10-09T17:08:06+00:00", + "fieldMask": [ + "references[]" + ] + }, + "sourceTag": "github.com", + "summary": null, + "url": "https://github.com/python-social-auth/social-app-django/pull/803" + }, + { + "kind": "advisory", + "provenance": { + "source": "ghsa", + "kind": "reference", + "value": "https://github.com/python-social-auth/social-app-django/security/advisories/GHSA-wv4w-6qv2-qqfg", + "decisionReason": null, + "recordedAt": "2025-10-09T17:08:06+00:00", + "fieldMask": [ + "references[]" + ] + }, + "sourceTag": "github.com", + "summary": null, + "url": "https://github.com/python-social-auth/social-app-django/security/advisories/GHSA-wv4w-6qv2-qqfg" + } + ], + "severity": "medium", + "summary": "### Impact\n\nUpon authentication, the user could be associated by e-mail even if the `associate_by_email` pipeline was not included. This could lead to account compromise when a third-party authentication service does not validate provided e-mail addresses or doesn't require unique e-mail addresses.\n\n### Patches\n\n* https://github.com/python-social-auth/social-app-django/pull/803\n\n### Workarounds\n\nReview the authentication service policy on e-mail addresses; many will not allow exploiting this vulnerability.", + "title": "Python Social Auth - Django has unsafe account association" + } +] \ No newline at end of file diff --git a/src/StellaOps.Feedser.Source.Osv.Tests/Fixtures/osv-ghsa.osv.json b/src/StellaOps.Feedser.Source.Osv.Tests/Fixtures/osv-ghsa.osv.json new file mode 100644 index 00000000..f9bf4b43 --- /dev/null +++ b/src/StellaOps.Feedser.Source.Osv.Tests/Fixtures/osv-ghsa.osv.json @@ -0,0 +1,3363 @@ +[ + { + "advisoryKey": "GHSA-77vh-xpmg-72qh", + "affectedPackages": [ + { + "type": "semver", + "identifier": "pkg:golang/github.com/opencontainers/image-spec", + "platform": "Go", + "versionRanges": [ + { + "fixedVersion": "1.0.2", + "introducedVersion": "0", + "lastAffectedVersion": null, + "primitives": { + "evr": null, + "hasVendorExtensions": false, + "nevra": null, + "semVer": { + "style": "range", + "introduced": "0", + "introducedInclusive": true, + "fixed": "1.0.2", + "fixedInclusive": false, + "lastAffected": null, + "lastAffectedInclusive": true, + "exactValue": null, + "constraintExpression": ">=0 <1.0.2" + }, + "vendorExtensions": null + }, + "provenance": { + "source": "osv", + "kind": "range", + "value": "pkg:golang/github.com/opencontainers/image-spec", + "decisionReason": "source-default", + "recordedAt": "2021-11-24T19:43:35+00:00", + "fieldMask": [] + }, + "rangeExpression": ">=0 <1.0.2", + "rangeKind": "semver" + } + ], + "normalizedVersions": [ + { + "scheme": "semver", + "type": "range", + "min": "0", + "minInclusive": true, + "max": "1.0.2", + "maxInclusive": false, + "value": null, + "notes": "osv:Go:GHSA-77vh-xpmg-72qh:pkg:golang/github.com/opencontainers/image-spec" + } + ], + "statuses": [], + "provenance": [ + { + "source": "osv", + "kind": "affected", + "value": "pkg:golang/github.com/opencontainers/image-spec", + "decisionReason": "source-default", + "recordedAt": "2021-11-24T19:43:35+00:00", + "fieldMask": [] + } + ] + } + ], + "aliases": [ + "CGA-j36r-723f-8c29", + "GHSA-77vh-xpmg-72qh" + ], + "cvssMetrics": [ + { + "baseScore": 3, + "baseSeverity": "low", + "provenance": { + "source": "osv", + "kind": "cvss", + "value": "CVSS_V3", + "decisionReason": "source-default", + "recordedAt": "2021-11-24T19:43:35+00:00", + "fieldMask": [] + }, + "vector": "CVSS:3.1/AV:N/AC:H/PR:L/UI:R/S:C/C:N/I:L/A:N", + "version": "3.1" + } + ], + "exploitKnown": false, + "language": "en", + "modified": "2021-11-24T19:43:35+00:00", + "provenance": [ + { + "source": "osv", + "kind": "document", + "value": "https://osv.dev/vulnerability/GHSA-77vh-xpmg-72qh", + "decisionReason": "source-default", + "recordedAt": "2021-11-18T16:02:41+00:00", + "fieldMask": [] + }, + { + "source": "osv", + "kind": "mapping", + "value": "GHSA-77vh-xpmg-72qh", + "decisionReason": "source-default", + "recordedAt": "2021-11-24T19:43:35+00:00", + "fieldMask": [] + } + ], + "published": "2021-11-18T16:02:41+00:00", + "references": [ + { + "kind": null, + "provenance": { + "source": "osv", + "kind": "reference", + "value": "https://github.com/opencontainers/distribution-spec/security/advisories/GHSA-mc8v-mgrf-8f4m", + "decisionReason": "source-default", + "recordedAt": "2021-11-24T19:43:35+00:00", + "fieldMask": [] + }, + "sourceTag": "WEB", + "summary": null, + "url": "https://github.com/opencontainers/distribution-spec/security/advisories/GHSA-mc8v-mgrf-8f4m" + }, + { + "kind": null, + "provenance": { + "source": "osv", + "kind": "reference", + "value": "https://github.com/opencontainers/image-spec", + "decisionReason": "source-default", + "recordedAt": "2021-11-24T19:43:35+00:00", + "fieldMask": [] + }, + "sourceTag": "PACKAGE", + "summary": null, + "url": "https://github.com/opencontainers/image-spec" + }, + { + "kind": null, + "provenance": { + "source": "osv", + "kind": "reference", + "value": "https://github.com/opencontainers/image-spec/commit/693428a734f5bab1a84bd2f990d92ef1111cd60c", + "decisionReason": "source-default", + "recordedAt": "2021-11-24T19:43:35+00:00", + "fieldMask": [] + }, + "sourceTag": "WEB", + "summary": null, + "url": "https://github.com/opencontainers/image-spec/commit/693428a734f5bab1a84bd2f990d92ef1111cd60c" + }, + { + "kind": null, + "provenance": { + "source": "osv", + "kind": "reference", + "value": "https://github.com/opencontainers/image-spec/releases/tag/v1.0.2", + "decisionReason": "source-default", + "recordedAt": "2021-11-24T19:43:35+00:00", + "fieldMask": [] + }, + "sourceTag": "WEB", + "summary": null, + "url": "https://github.com/opencontainers/image-spec/releases/tag/v1.0.2" + }, + { + "kind": null, + "provenance": { + "source": "osv", + "kind": "reference", + "value": "https://github.com/opencontainers/image-spec/security/advisories/GHSA-77vh-xpmg-72qh", + "decisionReason": "source-default", + "recordedAt": "2021-11-24T19:43:35+00:00", + "fieldMask": [] + }, + "sourceTag": "WEB", + "summary": null, + "url": "https://github.com/opencontainers/image-spec/security/advisories/GHSA-77vh-xpmg-72qh" + } + ], + "severity": "low", + "summary": "### Impact In the OCI Image Specification version 1.0.1 and prior, manifest and index documents are not self-describing and documents with a single digest could be interpreted as either a manifest or an index. ### Patches The Image Specification will be updated to recommend that both manifest and index documents contain a `mediaType` field to identify the type of document. Release [v1.0.2](https://github.com/opencontainers/image-spec/releases/tag/v1.0.2) includes these updates. ### Workarounds Software attempting to deserialize an ambiguous document may reject the document if it contains both “manifests” and “layers” fields or “manifests” and “config” fields. ### References https://github.com/opencontainers/distribution-spec/security/advisories/GHSA-mc8v-mgrf-8f4m ### For more information If you have any questions or comments about this advisory: * Open an issue in https://github.com/opencontainers/image-spec * Email us at [security@opencontainers.org](mailto:security@opencontainers.org) * https://github.com/opencontainers/image-spec/commits/v1.0.2", + "title": "Clarify `mediaType` handling" + }, + { + "advisoryKey": "GHSA-7rjr-3q55-vv33", + "affectedPackages": [ + { + "type": "semver", + "identifier": "pkg:maven/org.apache.logging.log4j/log4j-core", + "platform": "Maven", + "versionRanges": [ + { + "fixedVersion": "2.16.0", + "introducedVersion": "2.13.0", + "lastAffectedVersion": null, + "primitives": { + "evr": null, + "hasVendorExtensions": false, + "nevra": null, + "semVer": { + "style": "range", + "introduced": "2.13.0", + "introducedInclusive": true, + "fixed": "2.16.0", + "fixedInclusive": false, + "lastAffected": null, + "lastAffectedInclusive": true, + "exactValue": null, + "constraintExpression": ">=2.13.0 <2.16.0" + }, + "vendorExtensions": null + }, + "provenance": { + "source": "osv", + "kind": "range", + "value": "pkg:maven/org.apache.logging.log4j/log4j-core", + "decisionReason": "source-default", + "recordedAt": "2025-05-09T13:13:16.169374+00:00", + "fieldMask": [] + }, + "rangeExpression": ">=2.13.0 <2.16.0", + "rangeKind": "semver" + } + ], + "normalizedVersions": [ + { + "scheme": "semver", + "type": "exact", + "min": null, + "minInclusive": null, + "max": null, + "maxInclusive": null, + "value": "2.13.0", + "notes": "osv:Maven:GHSA-7rjr-3q55-vv33:pkg:maven/org.apache.logging.log4j/log4j-core" + }, + { + "scheme": "semver", + "type": "exact", + "min": null, + "minInclusive": null, + "max": null, + "maxInclusive": null, + "value": "2.13.1", + "notes": "osv:Maven:GHSA-7rjr-3q55-vv33:pkg:maven/org.apache.logging.log4j/log4j-core" + }, + { + "scheme": "semver", + "type": "exact", + "min": null, + "minInclusive": null, + "max": null, + "maxInclusive": null, + "value": "2.13.2", + "notes": "osv:Maven:GHSA-7rjr-3q55-vv33:pkg:maven/org.apache.logging.log4j/log4j-core" + }, + { + "scheme": "semver", + "type": "exact", + "min": null, + "minInclusive": null, + "max": null, + "maxInclusive": null, + "value": "2.13.3", + "notes": "osv:Maven:GHSA-7rjr-3q55-vv33:pkg:maven/org.apache.logging.log4j/log4j-core" + }, + { + "scheme": "semver", + "type": "exact", + "min": null, + "minInclusive": null, + "max": null, + "maxInclusive": null, + "value": "2.14.0", + "notes": "osv:Maven:GHSA-7rjr-3q55-vv33:pkg:maven/org.apache.logging.log4j/log4j-core" + }, + { + "scheme": "semver", + "type": "exact", + "min": null, + "minInclusive": null, + "max": null, + "maxInclusive": null, + "value": "2.14.1", + "notes": "osv:Maven:GHSA-7rjr-3q55-vv33:pkg:maven/org.apache.logging.log4j/log4j-core" + }, + { + "scheme": "semver", + "type": "exact", + "min": null, + "minInclusive": null, + "max": null, + "maxInclusive": null, + "value": "2.15.0", + "notes": "osv:Maven:GHSA-7rjr-3q55-vv33:pkg:maven/org.apache.logging.log4j/log4j-core" + }, + { + "scheme": "semver", + "type": "range", + "min": "2.13.0", + "minInclusive": true, + "max": "2.16.0", + "maxInclusive": false, + "value": null, + "notes": "osv:Maven:GHSA-7rjr-3q55-vv33:pkg:maven/org.apache.logging.log4j/log4j-core" + } + ], + "statuses": [], + "provenance": [ + { + "source": "osv", + "kind": "affected", + "value": "pkg:maven/org.apache.logging.log4j/log4j-core", + "decisionReason": "source-default", + "recordedAt": "2025-05-09T13:13:16.169374+00:00", + "fieldMask": [] + } + ] + }, + { + "type": "semver", + "identifier": "pkg:maven/org.apache.logging.log4j/log4j-core", + "platform": "Maven", + "versionRanges": [ + { + "fixedVersion": "2.12.2", + "introducedVersion": "0", + "lastAffectedVersion": null, + "primitives": { + "evr": null, + "hasVendorExtensions": false, + "nevra": null, + "semVer": { + "style": "range", + "introduced": "0", + "introducedInclusive": true, + "fixed": "2.12.2", + "fixedInclusive": false, + "lastAffected": null, + "lastAffectedInclusive": true, + "exactValue": null, + "constraintExpression": ">=0 <2.12.2" + }, + "vendorExtensions": null + }, + "provenance": { + "source": "osv", + "kind": "range", + "value": "pkg:maven/org.apache.logging.log4j/log4j-core", + "decisionReason": "source-default", + "recordedAt": "2025-05-09T13:13:16.169374+00:00", + "fieldMask": [] + }, + "rangeExpression": ">=0 <2.12.2", + "rangeKind": "semver" + } + ], + "normalizedVersions": [ + { + "scheme": "semver", + "type": "exact", + "min": null, + "minInclusive": null, + "max": null, + "maxInclusive": null, + "value": "2.0", + "notes": "osv:Maven:GHSA-7rjr-3q55-vv33:pkg:maven/org.apache.logging.log4j/log4j-core" + }, + { + "scheme": "semver", + "type": "exact", + "min": null, + "minInclusive": null, + "max": null, + "maxInclusive": null, + "value": "2.0-alpha1", + "notes": "osv:Maven:GHSA-7rjr-3q55-vv33:pkg:maven/org.apache.logging.log4j/log4j-core" + }, + { + "scheme": "semver", + "type": "exact", + "min": null, + "minInclusive": null, + "max": null, + "maxInclusive": null, + "value": "2.0-alpha2", + "notes": "osv:Maven:GHSA-7rjr-3q55-vv33:pkg:maven/org.apache.logging.log4j/log4j-core" + }, + { + "scheme": "semver", + "type": "exact", + "min": null, + "minInclusive": null, + "max": null, + "maxInclusive": null, + "value": "2.0-beta1", + "notes": "osv:Maven:GHSA-7rjr-3q55-vv33:pkg:maven/org.apache.logging.log4j/log4j-core" + }, + { + "scheme": "semver", + "type": "exact", + "min": null, + "minInclusive": null, + "max": null, + "maxInclusive": null, + "value": "2.0-beta2", + "notes": "osv:Maven:GHSA-7rjr-3q55-vv33:pkg:maven/org.apache.logging.log4j/log4j-core" + }, + { + "scheme": "semver", + "type": "exact", + "min": null, + "minInclusive": null, + "max": null, + "maxInclusive": null, + "value": "2.0-beta3", + "notes": "osv:Maven:GHSA-7rjr-3q55-vv33:pkg:maven/org.apache.logging.log4j/log4j-core" + }, + { + "scheme": "semver", + "type": "exact", + "min": null, + "minInclusive": null, + "max": null, + "maxInclusive": null, + "value": "2.0-beta4", + "notes": "osv:Maven:GHSA-7rjr-3q55-vv33:pkg:maven/org.apache.logging.log4j/log4j-core" + }, + { + "scheme": "semver", + "type": "exact", + "min": null, + "minInclusive": null, + "max": null, + "maxInclusive": null, + "value": "2.0-beta5", + "notes": "osv:Maven:GHSA-7rjr-3q55-vv33:pkg:maven/org.apache.logging.log4j/log4j-core" + }, + { + "scheme": "semver", + "type": "exact", + "min": null, + "minInclusive": null, + "max": null, + "maxInclusive": null, + "value": "2.0-beta6", + "notes": "osv:Maven:GHSA-7rjr-3q55-vv33:pkg:maven/org.apache.logging.log4j/log4j-core" + }, + { + "scheme": "semver", + "type": "exact", + "min": null, + "minInclusive": null, + "max": null, + "maxInclusive": null, + "value": "2.0-beta7", + "notes": "osv:Maven:GHSA-7rjr-3q55-vv33:pkg:maven/org.apache.logging.log4j/log4j-core" + }, + { + "scheme": "semver", + "type": "exact", + "min": null, + "minInclusive": null, + "max": null, + "maxInclusive": null, + "value": "2.0-beta8", + "notes": "osv:Maven:GHSA-7rjr-3q55-vv33:pkg:maven/org.apache.logging.log4j/log4j-core" + }, + { + "scheme": "semver", + "type": "exact", + "min": null, + "minInclusive": null, + "max": null, + "maxInclusive": null, + "value": "2.0-beta9", + "notes": "osv:Maven:GHSA-7rjr-3q55-vv33:pkg:maven/org.apache.logging.log4j/log4j-core" + }, + { + "scheme": "semver", + "type": "exact", + "min": null, + "minInclusive": null, + "max": null, + "maxInclusive": null, + "value": "2.0-rc1", + "notes": "osv:Maven:GHSA-7rjr-3q55-vv33:pkg:maven/org.apache.logging.log4j/log4j-core" + }, + { + "scheme": "semver", + "type": "exact", + "min": null, + "minInclusive": null, + "max": null, + "maxInclusive": null, + "value": "2.0-rc2", + "notes": "osv:Maven:GHSA-7rjr-3q55-vv33:pkg:maven/org.apache.logging.log4j/log4j-core" + }, + { + "scheme": "semver", + "type": "exact", + "min": null, + "minInclusive": null, + "max": null, + "maxInclusive": null, + "value": "2.0.1", + "notes": "osv:Maven:GHSA-7rjr-3q55-vv33:pkg:maven/org.apache.logging.log4j/log4j-core" + }, + { + "scheme": "semver", + "type": "exact", + "min": null, + "minInclusive": null, + "max": null, + "maxInclusive": null, + "value": "2.0.2", + "notes": "osv:Maven:GHSA-7rjr-3q55-vv33:pkg:maven/org.apache.logging.log4j/log4j-core" + }, + { + "scheme": "semver", + "type": "exact", + "min": null, + "minInclusive": null, + "max": null, + "maxInclusive": null, + "value": "2.1", + "notes": "osv:Maven:GHSA-7rjr-3q55-vv33:pkg:maven/org.apache.logging.log4j/log4j-core" + }, + { + "scheme": "semver", + "type": "exact", + "min": null, + "minInclusive": null, + "max": null, + "maxInclusive": null, + "value": "2.10.0", + "notes": "osv:Maven:GHSA-7rjr-3q55-vv33:pkg:maven/org.apache.logging.log4j/log4j-core" + }, + { + "scheme": "semver", + "type": "exact", + "min": null, + "minInclusive": null, + "max": null, + "maxInclusive": null, + "value": "2.11.0", + "notes": "osv:Maven:GHSA-7rjr-3q55-vv33:pkg:maven/org.apache.logging.log4j/log4j-core" + }, + { + "scheme": "semver", + "type": "exact", + "min": null, + "minInclusive": null, + "max": null, + "maxInclusive": null, + "value": "2.11.1", + "notes": "osv:Maven:GHSA-7rjr-3q55-vv33:pkg:maven/org.apache.logging.log4j/log4j-core" + }, + { + "scheme": "semver", + "type": "exact", + "min": null, + "minInclusive": null, + "max": null, + "maxInclusive": null, + "value": "2.11.2", + "notes": "osv:Maven:GHSA-7rjr-3q55-vv33:pkg:maven/org.apache.logging.log4j/log4j-core" + }, + { + "scheme": "semver", + "type": "exact", + "min": null, + "minInclusive": null, + "max": null, + "maxInclusive": null, + "value": "2.12.0", + "notes": "osv:Maven:GHSA-7rjr-3q55-vv33:pkg:maven/org.apache.logging.log4j/log4j-core" + }, + { + "scheme": "semver", + "type": "exact", + "min": null, + "minInclusive": null, + "max": null, + "maxInclusive": null, + "value": "2.12.1", + "notes": "osv:Maven:GHSA-7rjr-3q55-vv33:pkg:maven/org.apache.logging.log4j/log4j-core" + }, + { + "scheme": "semver", + "type": "exact", + "min": null, + "minInclusive": null, + "max": null, + "maxInclusive": null, + "value": "2.2", + "notes": "osv:Maven:GHSA-7rjr-3q55-vv33:pkg:maven/org.apache.logging.log4j/log4j-core" + }, + { + "scheme": "semver", + "type": "exact", + "min": null, + "minInclusive": null, + "max": null, + "maxInclusive": null, + "value": "2.3", + "notes": "osv:Maven:GHSA-7rjr-3q55-vv33:pkg:maven/org.apache.logging.log4j/log4j-core" + }, + { + "scheme": "semver", + "type": "exact", + "min": null, + "minInclusive": null, + "max": null, + "maxInclusive": null, + "value": "2.3.1", + "notes": "osv:Maven:GHSA-7rjr-3q55-vv33:pkg:maven/org.apache.logging.log4j/log4j-core" + }, + { + "scheme": "semver", + "type": "exact", + "min": null, + "minInclusive": null, + "max": null, + "maxInclusive": null, + "value": "2.3.2", + "notes": "osv:Maven:GHSA-7rjr-3q55-vv33:pkg:maven/org.apache.logging.log4j/log4j-core" + }, + { + "scheme": "semver", + "type": "exact", + "min": null, + "minInclusive": null, + "max": null, + "maxInclusive": null, + "value": "2.4", + "notes": "osv:Maven:GHSA-7rjr-3q55-vv33:pkg:maven/org.apache.logging.log4j/log4j-core" + }, + { + "scheme": "semver", + "type": "exact", + "min": null, + "minInclusive": null, + "max": null, + "maxInclusive": null, + "value": "2.4.1", + "notes": "osv:Maven:GHSA-7rjr-3q55-vv33:pkg:maven/org.apache.logging.log4j/log4j-core" + }, + { + "scheme": "semver", + "type": "exact", + "min": null, + "minInclusive": null, + "max": null, + "maxInclusive": null, + "value": "2.5", + "notes": "osv:Maven:GHSA-7rjr-3q55-vv33:pkg:maven/org.apache.logging.log4j/log4j-core" + }, + { + "scheme": "semver", + "type": "exact", + "min": null, + "minInclusive": null, + "max": null, + "maxInclusive": null, + "value": "2.6", + "notes": "osv:Maven:GHSA-7rjr-3q55-vv33:pkg:maven/org.apache.logging.log4j/log4j-core" + }, + { + "scheme": "semver", + "type": "exact", + "min": null, + "minInclusive": null, + "max": null, + "maxInclusive": null, + "value": "2.6.1", + "notes": "osv:Maven:GHSA-7rjr-3q55-vv33:pkg:maven/org.apache.logging.log4j/log4j-core" + }, + { + "scheme": "semver", + "type": "exact", + "min": null, + "minInclusive": null, + "max": null, + "maxInclusive": null, + "value": "2.6.2", + "notes": "osv:Maven:GHSA-7rjr-3q55-vv33:pkg:maven/org.apache.logging.log4j/log4j-core" + }, + { + "scheme": "semver", + "type": "exact", + "min": null, + "minInclusive": null, + "max": null, + "maxInclusive": null, + "value": "2.7", + "notes": "osv:Maven:GHSA-7rjr-3q55-vv33:pkg:maven/org.apache.logging.log4j/log4j-core" + }, + { + "scheme": "semver", + "type": "exact", + "min": null, + "minInclusive": null, + "max": null, + "maxInclusive": null, + "value": "2.8", + "notes": "osv:Maven:GHSA-7rjr-3q55-vv33:pkg:maven/org.apache.logging.log4j/log4j-core" + }, + { + "scheme": "semver", + "type": "exact", + "min": null, + "minInclusive": null, + "max": null, + "maxInclusive": null, + "value": "2.8.1", + "notes": "osv:Maven:GHSA-7rjr-3q55-vv33:pkg:maven/org.apache.logging.log4j/log4j-core" + }, + { + "scheme": "semver", + "type": "exact", + "min": null, + "minInclusive": null, + "max": null, + "maxInclusive": null, + "value": "2.8.2", + "notes": "osv:Maven:GHSA-7rjr-3q55-vv33:pkg:maven/org.apache.logging.log4j/log4j-core" + }, + { + "scheme": "semver", + "type": "exact", + "min": null, + "minInclusive": null, + "max": null, + "maxInclusive": null, + "value": "2.9.0", + "notes": "osv:Maven:GHSA-7rjr-3q55-vv33:pkg:maven/org.apache.logging.log4j/log4j-core" + }, + { + "scheme": "semver", + "type": "exact", + "min": null, + "minInclusive": null, + "max": null, + "maxInclusive": null, + "value": "2.9.1", + "notes": "osv:Maven:GHSA-7rjr-3q55-vv33:pkg:maven/org.apache.logging.log4j/log4j-core" + }, + { + "scheme": "semver", + "type": "range", + "min": "0", + "minInclusive": true, + "max": "2.12.2", + "maxInclusive": false, + "value": null, + "notes": "osv:Maven:GHSA-7rjr-3q55-vv33:pkg:maven/org.apache.logging.log4j/log4j-core" + } + ], + "statuses": [], + "provenance": [ + { + "source": "osv", + "kind": "affected", + "value": "pkg:maven/org.apache.logging.log4j/log4j-core", + "decisionReason": "source-default", + "recordedAt": "2025-05-09T13:13:16.169374+00:00", + "fieldMask": [] + } + ] + }, + { + "type": "semver", + "identifier": "pkg:maven/org.ops4j.pax.logging/pax-logging-log4j2", + "platform": "Maven", + "versionRanges": [ + { + "fixedVersion": "1.9.2", + "introducedVersion": "1.8.0", + "lastAffectedVersion": null, + "primitives": { + "evr": null, + "hasVendorExtensions": false, + "nevra": null, + "semVer": { + "style": "range", + "introduced": "1.8.0", + "introducedInclusive": true, + "fixed": "1.9.2", + "fixedInclusive": false, + "lastAffected": null, + "lastAffectedInclusive": true, + "exactValue": null, + "constraintExpression": ">=1.8.0 <1.9.2" + }, + "vendorExtensions": null + }, + "provenance": { + "source": "osv", + "kind": "range", + "value": "pkg:maven/org.ops4j.pax.logging/pax-logging-log4j2", + "decisionReason": "source-default", + "recordedAt": "2025-05-09T13:13:16.169374+00:00", + "fieldMask": [] + }, + "rangeExpression": ">=1.8.0 <1.9.2", + "rangeKind": "semver" + } + ], + "normalizedVersions": [ + { + "scheme": "semver", + "type": "exact", + "min": null, + "minInclusive": null, + "max": null, + "maxInclusive": null, + "value": "1.8.0", + "notes": "osv:Maven:GHSA-7rjr-3q55-vv33:pkg:maven/org.ops4j.pax.logging/pax-logging-log4j2" + }, + { + "scheme": "semver", + "type": "exact", + "min": null, + "minInclusive": null, + "max": null, + "maxInclusive": null, + "value": "1.8.1", + "notes": "osv:Maven:GHSA-7rjr-3q55-vv33:pkg:maven/org.ops4j.pax.logging/pax-logging-log4j2" + }, + { + "scheme": "semver", + "type": "exact", + "min": null, + "minInclusive": null, + "max": null, + "maxInclusive": null, + "value": "1.8.2", + "notes": "osv:Maven:GHSA-7rjr-3q55-vv33:pkg:maven/org.ops4j.pax.logging/pax-logging-log4j2" + }, + { + "scheme": "semver", + "type": "exact", + "min": null, + "minInclusive": null, + "max": null, + "maxInclusive": null, + "value": "1.8.3", + "notes": "osv:Maven:GHSA-7rjr-3q55-vv33:pkg:maven/org.ops4j.pax.logging/pax-logging-log4j2" + }, + { + "scheme": "semver", + "type": "exact", + "min": null, + "minInclusive": null, + "max": null, + "maxInclusive": null, + "value": "1.8.4", + "notes": "osv:Maven:GHSA-7rjr-3q55-vv33:pkg:maven/org.ops4j.pax.logging/pax-logging-log4j2" + }, + { + "scheme": "semver", + "type": "exact", + "min": null, + "minInclusive": null, + "max": null, + "maxInclusive": null, + "value": "1.8.5", + "notes": "osv:Maven:GHSA-7rjr-3q55-vv33:pkg:maven/org.ops4j.pax.logging/pax-logging-log4j2" + }, + { + "scheme": "semver", + "type": "exact", + "min": null, + "minInclusive": null, + "max": null, + "maxInclusive": null, + "value": "1.8.6", + "notes": "osv:Maven:GHSA-7rjr-3q55-vv33:pkg:maven/org.ops4j.pax.logging/pax-logging-log4j2" + }, + { + "scheme": "semver", + "type": "exact", + "min": null, + "minInclusive": null, + "max": null, + "maxInclusive": null, + "value": "1.8.7", + "notes": "osv:Maven:GHSA-7rjr-3q55-vv33:pkg:maven/org.ops4j.pax.logging/pax-logging-log4j2" + }, + { + "scheme": "semver", + "type": "exact", + "min": null, + "minInclusive": null, + "max": null, + "maxInclusive": null, + "value": "1.9.0", + "notes": "osv:Maven:GHSA-7rjr-3q55-vv33:pkg:maven/org.ops4j.pax.logging/pax-logging-log4j2" + }, + { + "scheme": "semver", + "type": "exact", + "min": null, + "minInclusive": null, + "max": null, + "maxInclusive": null, + "value": "1.9.1", + "notes": "osv:Maven:GHSA-7rjr-3q55-vv33:pkg:maven/org.ops4j.pax.logging/pax-logging-log4j2" + }, + { + "scheme": "semver", + "type": "range", + "min": "1.8.0", + "minInclusive": true, + "max": "1.9.2", + "maxInclusive": false, + "value": null, + "notes": "osv:Maven:GHSA-7rjr-3q55-vv33:pkg:maven/org.ops4j.pax.logging/pax-logging-log4j2" + } + ], + "statuses": [], + "provenance": [ + { + "source": "osv", + "kind": "affected", + "value": "pkg:maven/org.ops4j.pax.logging/pax-logging-log4j2", + "decisionReason": "source-default", + "recordedAt": "2025-05-09T13:13:16.169374+00:00", + "fieldMask": [] + } + ] + }, + { + "type": "semver", + "identifier": "pkg:maven/org.ops4j.pax.logging/pax-logging-log4j2", + "platform": "Maven", + "versionRanges": [ + { + "fixedVersion": "1.10.8", + "introducedVersion": "1.10.0", + "lastAffectedVersion": null, + "primitives": { + "evr": null, + "hasVendorExtensions": false, + "nevra": null, + "semVer": { + "style": "range", + "introduced": "1.10.0", + "introducedInclusive": true, + "fixed": "1.10.8", + "fixedInclusive": false, + "lastAffected": null, + "lastAffectedInclusive": true, + "exactValue": null, + "constraintExpression": ">=1.10.0 <1.10.8" + }, + "vendorExtensions": null + }, + "provenance": { + "source": "osv", + "kind": "range", + "value": "pkg:maven/org.ops4j.pax.logging/pax-logging-log4j2", + "decisionReason": "source-default", + "recordedAt": "2025-05-09T13:13:16.169374+00:00", + "fieldMask": [] + }, + "rangeExpression": ">=1.10.0 <1.10.8", + "rangeKind": "semver" + } + ], + "normalizedVersions": [ + { + "scheme": "semver", + "type": "exact", + "min": null, + "minInclusive": null, + "max": null, + "maxInclusive": null, + "value": "1.10.0", + "notes": "osv:Maven:GHSA-7rjr-3q55-vv33:pkg:maven/org.ops4j.pax.logging/pax-logging-log4j2" + }, + { + "scheme": "semver", + "type": "exact", + "min": null, + "minInclusive": null, + "max": null, + "maxInclusive": null, + "value": "1.10.1", + "notes": "osv:Maven:GHSA-7rjr-3q55-vv33:pkg:maven/org.ops4j.pax.logging/pax-logging-log4j2" + }, + { + "scheme": "semver", + "type": "exact", + "min": null, + "minInclusive": null, + "max": null, + "maxInclusive": null, + "value": "1.10.2", + "notes": "osv:Maven:GHSA-7rjr-3q55-vv33:pkg:maven/org.ops4j.pax.logging/pax-logging-log4j2" + }, + { + "scheme": "semver", + "type": "exact", + "min": null, + "minInclusive": null, + "max": null, + "maxInclusive": null, + "value": "1.10.3", + "notes": "osv:Maven:GHSA-7rjr-3q55-vv33:pkg:maven/org.ops4j.pax.logging/pax-logging-log4j2" + }, + { + "scheme": "semver", + "type": "exact", + "min": null, + "minInclusive": null, + "max": null, + "maxInclusive": null, + "value": "1.10.4", + "notes": "osv:Maven:GHSA-7rjr-3q55-vv33:pkg:maven/org.ops4j.pax.logging/pax-logging-log4j2" + }, + { + "scheme": "semver", + "type": "exact", + "min": null, + "minInclusive": null, + "max": null, + "maxInclusive": null, + "value": "1.10.5", + "notes": "osv:Maven:GHSA-7rjr-3q55-vv33:pkg:maven/org.ops4j.pax.logging/pax-logging-log4j2" + }, + { + "scheme": "semver", + "type": "exact", + "min": null, + "minInclusive": null, + "max": null, + "maxInclusive": null, + "value": "1.10.6", + "notes": "osv:Maven:GHSA-7rjr-3q55-vv33:pkg:maven/org.ops4j.pax.logging/pax-logging-log4j2" + }, + { + "scheme": "semver", + "type": "exact", + "min": null, + "minInclusive": null, + "max": null, + "maxInclusive": null, + "value": "1.10.7", + "notes": "osv:Maven:GHSA-7rjr-3q55-vv33:pkg:maven/org.ops4j.pax.logging/pax-logging-log4j2" + }, + { + "scheme": "semver", + "type": "range", + "min": "1.10.0", + "minInclusive": true, + "max": "1.10.8", + "maxInclusive": false, + "value": null, + "notes": "osv:Maven:GHSA-7rjr-3q55-vv33:pkg:maven/org.ops4j.pax.logging/pax-logging-log4j2" + } + ], + "statuses": [], + "provenance": [ + { + "source": "osv", + "kind": "affected", + "value": "pkg:maven/org.ops4j.pax.logging/pax-logging-log4j2", + "decisionReason": "source-default", + "recordedAt": "2025-05-09T13:13:16.169374+00:00", + "fieldMask": [] + } + ] + }, + { + "type": "semver", + "identifier": "pkg:maven/org.ops4j.pax.logging/pax-logging-log4j2", + "platform": "Maven", + "versionRanges": [ + { + "fixedVersion": "1.11.11", + "introducedVersion": "1.11.0", + "lastAffectedVersion": null, + "primitives": { + "evr": null, + "hasVendorExtensions": false, + "nevra": null, + "semVer": { + "style": "range", + "introduced": "1.11.0", + "introducedInclusive": true, + "fixed": "1.11.11", + "fixedInclusive": false, + "lastAffected": null, + "lastAffectedInclusive": true, + "exactValue": null, + "constraintExpression": ">=1.11.0 <1.11.11" + }, + "vendorExtensions": null + }, + "provenance": { + "source": "osv", + "kind": "range", + "value": "pkg:maven/org.ops4j.pax.logging/pax-logging-log4j2", + "decisionReason": "source-default", + "recordedAt": "2025-05-09T13:13:16.169374+00:00", + "fieldMask": [] + }, + "rangeExpression": ">=1.11.0 <1.11.11", + "rangeKind": "semver" + } + ], + "normalizedVersions": [ + { + "scheme": "semver", + "type": "exact", + "min": null, + "minInclusive": null, + "max": null, + "maxInclusive": null, + "value": "1.11.0", + "notes": "osv:Maven:GHSA-7rjr-3q55-vv33:pkg:maven/org.ops4j.pax.logging/pax-logging-log4j2" + }, + { + "scheme": "semver", + "type": "exact", + "min": null, + "minInclusive": null, + "max": null, + "maxInclusive": null, + "value": "1.11.1", + "notes": "osv:Maven:GHSA-7rjr-3q55-vv33:pkg:maven/org.ops4j.pax.logging/pax-logging-log4j2" + }, + { + "scheme": "semver", + "type": "exact", + "min": null, + "minInclusive": null, + "max": null, + "maxInclusive": null, + "value": "1.11.10", + "notes": "osv:Maven:GHSA-7rjr-3q55-vv33:pkg:maven/org.ops4j.pax.logging/pax-logging-log4j2" + }, + { + "scheme": "semver", + "type": "exact", + "min": null, + "minInclusive": null, + "max": null, + "maxInclusive": null, + "value": "1.11.2", + "notes": "osv:Maven:GHSA-7rjr-3q55-vv33:pkg:maven/org.ops4j.pax.logging/pax-logging-log4j2" + }, + { + "scheme": "semver", + "type": "exact", + "min": null, + "minInclusive": null, + "max": null, + "maxInclusive": null, + "value": "1.11.3", + "notes": "osv:Maven:GHSA-7rjr-3q55-vv33:pkg:maven/org.ops4j.pax.logging/pax-logging-log4j2" + }, + { + "scheme": "semver", + "type": "exact", + "min": null, + "minInclusive": null, + "max": null, + "maxInclusive": null, + "value": "1.11.4", + "notes": "osv:Maven:GHSA-7rjr-3q55-vv33:pkg:maven/org.ops4j.pax.logging/pax-logging-log4j2" + }, + { + "scheme": "semver", + "type": "exact", + "min": null, + "minInclusive": null, + "max": null, + "maxInclusive": null, + "value": "1.11.5", + "notes": "osv:Maven:GHSA-7rjr-3q55-vv33:pkg:maven/org.ops4j.pax.logging/pax-logging-log4j2" + }, + { + "scheme": "semver", + "type": "exact", + "min": null, + "minInclusive": null, + "max": null, + "maxInclusive": null, + "value": "1.11.6", + "notes": "osv:Maven:GHSA-7rjr-3q55-vv33:pkg:maven/org.ops4j.pax.logging/pax-logging-log4j2" + }, + { + "scheme": "semver", + "type": "exact", + "min": null, + "minInclusive": null, + "max": null, + "maxInclusive": null, + "value": "1.11.7", + "notes": "osv:Maven:GHSA-7rjr-3q55-vv33:pkg:maven/org.ops4j.pax.logging/pax-logging-log4j2" + }, + { + "scheme": "semver", + "type": "exact", + "min": null, + "minInclusive": null, + "max": null, + "maxInclusive": null, + "value": "1.11.8", + "notes": "osv:Maven:GHSA-7rjr-3q55-vv33:pkg:maven/org.ops4j.pax.logging/pax-logging-log4j2" + }, + { + "scheme": "semver", + "type": "exact", + "min": null, + "minInclusive": null, + "max": null, + "maxInclusive": null, + "value": "1.11.9", + "notes": "osv:Maven:GHSA-7rjr-3q55-vv33:pkg:maven/org.ops4j.pax.logging/pax-logging-log4j2" + }, + { + "scheme": "semver", + "type": "range", + "min": "1.11.0", + "minInclusive": true, + "max": "1.11.11", + "maxInclusive": false, + "value": null, + "notes": "osv:Maven:GHSA-7rjr-3q55-vv33:pkg:maven/org.ops4j.pax.logging/pax-logging-log4j2" + } + ], + "statuses": [], + "provenance": [ + { + "source": "osv", + "kind": "affected", + "value": "pkg:maven/org.ops4j.pax.logging/pax-logging-log4j2", + "decisionReason": "source-default", + "recordedAt": "2025-05-09T13:13:16.169374+00:00", + "fieldMask": [] + } + ] + }, + { + "type": "semver", + "identifier": "pkg:maven/org.ops4j.pax.logging/pax-logging-log4j2", + "platform": "Maven", + "versionRanges": [ + { + "fixedVersion": "2.0.12", + "introducedVersion": "2.0.0", + "lastAffectedVersion": null, + "primitives": { + "evr": null, + "hasVendorExtensions": false, + "nevra": null, + "semVer": { + "style": "range", + "introduced": "2.0.0", + "introducedInclusive": true, + "fixed": "2.0.12", + "fixedInclusive": false, + "lastAffected": null, + "lastAffectedInclusive": true, + "exactValue": null, + "constraintExpression": ">=2.0.0 <2.0.12" + }, + "vendorExtensions": null + }, + "provenance": { + "source": "osv", + "kind": "range", + "value": "pkg:maven/org.ops4j.pax.logging/pax-logging-log4j2", + "decisionReason": "source-default", + "recordedAt": "2025-05-09T13:13:16.169374+00:00", + "fieldMask": [] + }, + "rangeExpression": ">=2.0.0 <2.0.12", + "rangeKind": "semver" + } + ], + "normalizedVersions": [ + { + "scheme": "semver", + "type": "exact", + "min": null, + "minInclusive": null, + "max": null, + "maxInclusive": null, + "value": "2.0.0", + "notes": "osv:Maven:GHSA-7rjr-3q55-vv33:pkg:maven/org.ops4j.pax.logging/pax-logging-log4j2" + }, + { + "scheme": "semver", + "type": "exact", + "min": null, + "minInclusive": null, + "max": null, + "maxInclusive": null, + "value": "2.0.1", + "notes": "osv:Maven:GHSA-7rjr-3q55-vv33:pkg:maven/org.ops4j.pax.logging/pax-logging-log4j2" + }, + { + "scheme": "semver", + "type": "exact", + "min": null, + "minInclusive": null, + "max": null, + "maxInclusive": null, + "value": "2.0.10", + "notes": "osv:Maven:GHSA-7rjr-3q55-vv33:pkg:maven/org.ops4j.pax.logging/pax-logging-log4j2" + }, + { + "scheme": "semver", + "type": "exact", + "min": null, + "minInclusive": null, + "max": null, + "maxInclusive": null, + "value": "2.0.11", + "notes": "osv:Maven:GHSA-7rjr-3q55-vv33:pkg:maven/org.ops4j.pax.logging/pax-logging-log4j2" + }, + { + "scheme": "semver", + "type": "exact", + "min": null, + "minInclusive": null, + "max": null, + "maxInclusive": null, + "value": "2.0.2", + "notes": "osv:Maven:GHSA-7rjr-3q55-vv33:pkg:maven/org.ops4j.pax.logging/pax-logging-log4j2" + }, + { + "scheme": "semver", + "type": "exact", + "min": null, + "minInclusive": null, + "max": null, + "maxInclusive": null, + "value": "2.0.3", + "notes": "osv:Maven:GHSA-7rjr-3q55-vv33:pkg:maven/org.ops4j.pax.logging/pax-logging-log4j2" + }, + { + "scheme": "semver", + "type": "exact", + "min": null, + "minInclusive": null, + "max": null, + "maxInclusive": null, + "value": "2.0.4", + "notes": "osv:Maven:GHSA-7rjr-3q55-vv33:pkg:maven/org.ops4j.pax.logging/pax-logging-log4j2" + }, + { + "scheme": "semver", + "type": "exact", + "min": null, + "minInclusive": null, + "max": null, + "maxInclusive": null, + "value": "2.0.5", + "notes": "osv:Maven:GHSA-7rjr-3q55-vv33:pkg:maven/org.ops4j.pax.logging/pax-logging-log4j2" + }, + { + "scheme": "semver", + "type": "exact", + "min": null, + "minInclusive": null, + "max": null, + "maxInclusive": null, + "value": "2.0.6", + "notes": "osv:Maven:GHSA-7rjr-3q55-vv33:pkg:maven/org.ops4j.pax.logging/pax-logging-log4j2" + }, + { + "scheme": "semver", + "type": "exact", + "min": null, + "minInclusive": null, + "max": null, + "maxInclusive": null, + "value": "2.0.7", + "notes": "osv:Maven:GHSA-7rjr-3q55-vv33:pkg:maven/org.ops4j.pax.logging/pax-logging-log4j2" + }, + { + "scheme": "semver", + "type": "exact", + "min": null, + "minInclusive": null, + "max": null, + "maxInclusive": null, + "value": "2.0.8", + "notes": "osv:Maven:GHSA-7rjr-3q55-vv33:pkg:maven/org.ops4j.pax.logging/pax-logging-log4j2" + }, + { + "scheme": "semver", + "type": "exact", + "min": null, + "minInclusive": null, + "max": null, + "maxInclusive": null, + "value": "2.0.9", + "notes": "osv:Maven:GHSA-7rjr-3q55-vv33:pkg:maven/org.ops4j.pax.logging/pax-logging-log4j2" + }, + { + "scheme": "semver", + "type": "range", + "min": "2.0.0", + "minInclusive": true, + "max": "2.0.12", + "maxInclusive": false, + "value": null, + "notes": "osv:Maven:GHSA-7rjr-3q55-vv33:pkg:maven/org.ops4j.pax.logging/pax-logging-log4j2" + } + ], + "statuses": [], + "provenance": [ + { + "source": "osv", + "kind": "affected", + "value": "pkg:maven/org.ops4j.pax.logging/pax-logging-log4j2", + "decisionReason": "source-default", + "recordedAt": "2025-05-09T13:13:16.169374+00:00", + "fieldMask": [] + } + ] + } + ], + "aliases": [ + "CVE-2021-45046", + "GHSA-7rjr-3q55-vv33" + ], + "cvssMetrics": [ + { + "baseScore": 9, + "baseSeverity": "critical", + "provenance": { + "source": "osv", + "kind": "cvss", + "value": "CVSS_V3", + "decisionReason": "source-default", + "recordedAt": "2025-05-09T13:13:16.169374+00:00", + "fieldMask": [] + }, + "vector": "CVSS:3.1/AV:N/AC:H/PR:N/UI:N/S:C/C:H/I:H/A:H", + "version": "3.1" + } + ], + "exploitKnown": false, + "language": "en", + "modified": "2025-05-09T13:13:16.169374+00:00", + "provenance": [ + { + "source": "osv", + "kind": "document", + "value": "https://osv.dev/vulnerability/GHSA-7rjr-3q55-vv33", + "decisionReason": "source-default", + "recordedAt": "2021-12-14T18:01:28+00:00", + "fieldMask": [] + }, + { + "source": "osv", + "kind": "mapping", + "value": "GHSA-7rjr-3q55-vv33", + "decisionReason": "source-default", + "recordedAt": "2025-05-09T13:13:16.169374+00:00", + "fieldMask": [] + } + ], + "published": "2021-12-14T18:01:28+00:00", + "references": [ + { + "kind": null, + "provenance": { + "source": "osv", + "kind": "reference", + "value": "http://www.openwall.com/lists/oss-security/2021/12/14/4", + "decisionReason": "source-default", + "recordedAt": "2025-05-09T13:13:16.169374+00:00", + "fieldMask": [] + }, + "sourceTag": "WEB", + "summary": null, + "url": "http://www.openwall.com/lists/oss-security/2021/12/14/4" + }, + { + "kind": null, + "provenance": { + "source": "osv", + "kind": "reference", + "value": "http://www.openwall.com/lists/oss-security/2021/12/15/3", + "decisionReason": "source-default", + "recordedAt": "2025-05-09T13:13:16.169374+00:00", + "fieldMask": [] + }, + "sourceTag": "WEB", + "summary": null, + "url": "http://www.openwall.com/lists/oss-security/2021/12/15/3" + }, + { + "kind": null, + "provenance": { + "source": "osv", + "kind": "reference", + "value": "http://www.openwall.com/lists/oss-security/2021/12/18/1", + "decisionReason": "source-default", + "recordedAt": "2025-05-09T13:13:16.169374+00:00", + "fieldMask": [] + }, + "sourceTag": "WEB", + "summary": null, + "url": "http://www.openwall.com/lists/oss-security/2021/12/18/1" + }, + { + "kind": null, + "provenance": { + "source": "osv", + "kind": "reference", + "value": "https://cert-portal.siemens.com/productcert/pdf/ssa-397453.pdf", + "decisionReason": "source-default", + "recordedAt": "2025-05-09T13:13:16.169374+00:00", + "fieldMask": [] + }, + "sourceTag": "WEB", + "summary": null, + "url": "https://cert-portal.siemens.com/productcert/pdf/ssa-397453.pdf" + }, + { + "kind": null, + "provenance": { + "source": "osv", + "kind": "reference", + "value": "https://cert-portal.siemens.com/productcert/pdf/ssa-479842.pdf", + "decisionReason": "source-default", + "recordedAt": "2025-05-09T13:13:16.169374+00:00", + "fieldMask": [] + }, + "sourceTag": "WEB", + "summary": null, + "url": "https://cert-portal.siemens.com/productcert/pdf/ssa-479842.pdf" + }, + { + "kind": null, + "provenance": { + "source": "osv", + "kind": "reference", + "value": "https://cert-portal.siemens.com/productcert/pdf/ssa-661247.pdf", + "decisionReason": "source-default", + "recordedAt": "2025-05-09T13:13:16.169374+00:00", + "fieldMask": [] + }, + "sourceTag": "WEB", + "summary": null, + "url": "https://cert-portal.siemens.com/productcert/pdf/ssa-661247.pdf" + }, + { + "kind": null, + "provenance": { + "source": "osv", + "kind": "reference", + "value": "https://cert-portal.siemens.com/productcert/pdf/ssa-714170.pdf", + "decisionReason": "source-default", + "recordedAt": "2025-05-09T13:13:16.169374+00:00", + "fieldMask": [] + }, + "sourceTag": "WEB", + "summary": null, + "url": "https://cert-portal.siemens.com/productcert/pdf/ssa-714170.pdf" + }, + { + "kind": "advisory", + "provenance": { + "source": "osv", + "kind": "reference", + "value": "https://github.com/advisories/GHSA-jfh8-c2jp-5v3q", + "decisionReason": "source-default", + "recordedAt": "2025-05-09T13:13:16.169374+00:00", + "fieldMask": [] + }, + "sourceTag": "ADVISORY", + "summary": null, + "url": "https://github.com/advisories/GHSA-jfh8-c2jp-5v3q" + }, + { + "kind": null, + "provenance": { + "source": "osv", + "kind": "reference", + "value": "https://lists.fedoraproject.org/archives/list/package-announce@lists.fedoraproject.org/message/EOKPQGV24RRBBI4TBZUDQMM4MEH7MXCY", + "decisionReason": "source-default", + "recordedAt": "2025-05-09T13:13:16.169374+00:00", + "fieldMask": [] + }, + "sourceTag": "WEB", + "summary": null, + "url": "https://lists.fedoraproject.org/archives/list/package-announce@lists.fedoraproject.org/message/EOKPQGV24RRBBI4TBZUDQMM4MEH7MXCY" + }, + { + "kind": null, + "provenance": { + "source": "osv", + "kind": "reference", + "value": "https://lists.fedoraproject.org/archives/list/package-announce@lists.fedoraproject.org/message/SIG7FZULMNK2XF6FZRU4VWYDQXNMUGAJ", + "decisionReason": "source-default", + "recordedAt": "2025-05-09T13:13:16.169374+00:00", + "fieldMask": [] + }, + "sourceTag": "WEB", + "summary": null, + "url": "https://lists.fedoraproject.org/archives/list/package-announce@lists.fedoraproject.org/message/SIG7FZULMNK2XF6FZRU4VWYDQXNMUGAJ" + }, + { + "kind": null, + "provenance": { + "source": "osv", + "kind": "reference", + "value": "https://logging.apache.org/log4j/2.x/security.html", + "decisionReason": "source-default", + "recordedAt": "2025-05-09T13:13:16.169374+00:00", + "fieldMask": [] + }, + "sourceTag": "WEB", + "summary": null, + "url": "https://logging.apache.org/log4j/2.x/security.html" + }, + { + "kind": "advisory", + "provenance": { + "source": "osv", + "kind": "reference", + "value": "https://nvd.nist.gov/vuln/detail/CVE-2021-45046", + "decisionReason": "source-default", + "recordedAt": "2025-05-09T13:13:16.169374+00:00", + "fieldMask": [] + }, + "sourceTag": "ADVISORY", + "summary": null, + "url": "https://nvd.nist.gov/vuln/detail/CVE-2021-45046" + }, + { + "kind": null, + "provenance": { + "source": "osv", + "kind": "reference", + "value": "https://psirt.global.sonicwall.com/vuln-detail/SNWLID-2021-0032", + "decisionReason": "source-default", + "recordedAt": "2025-05-09T13:13:16.169374+00:00", + "fieldMask": [] + }, + "sourceTag": "WEB", + "summary": null, + "url": "https://psirt.global.sonicwall.com/vuln-detail/SNWLID-2021-0032" + }, + { + "kind": null, + "provenance": { + "source": "osv", + "kind": "reference", + "value": "https://sec.cloudapps.cisco.com/security/center/content/CiscoSecurityAdvisory/cisco-sa-apache-log4j-qRuKNEbd", + "decisionReason": "source-default", + "recordedAt": "2025-05-09T13:13:16.169374+00:00", + "fieldMask": [] + }, + "sourceTag": "WEB", + "summary": null, + "url": "https://sec.cloudapps.cisco.com/security/center/content/CiscoSecurityAdvisory/cisco-sa-apache-log4j-qRuKNEbd" + }, + { + "kind": null, + "provenance": { + "source": "osv", + "kind": "reference", + "value": "https://security.gentoo.org/glsa/202310-16", + "decisionReason": "source-default", + "recordedAt": "2025-05-09T13:13:16.169374+00:00", + "fieldMask": [] + }, + "sourceTag": "WEB", + "summary": null, + "url": "https://security.gentoo.org/glsa/202310-16" + }, + { + "kind": null, + "provenance": { + "source": "osv", + "kind": "reference", + "value": "https://www.cve.org/CVERecord?id=CVE-2021-44228", + "decisionReason": "source-default", + "recordedAt": "2025-05-09T13:13:16.169374+00:00", + "fieldMask": [] + }, + "sourceTag": "WEB", + "summary": null, + "url": "https://www.cve.org/CVERecord?id=CVE-2021-44228" + }, + { + "kind": null, + "provenance": { + "source": "osv", + "kind": "reference", + "value": "https://www.debian.org/security/2021/dsa-5022", + "decisionReason": "source-default", + "recordedAt": "2025-05-09T13:13:16.169374+00:00", + "fieldMask": [] + }, + "sourceTag": "WEB", + "summary": null, + "url": "https://www.debian.org/security/2021/dsa-5022" + }, + { + "kind": null, + "provenance": { + "source": "osv", + "kind": "reference", + "value": "https://www.intel.com/content/www/us/en/security-center/advisory/intel-sa-00646.html", + "decisionReason": "source-default", + "recordedAt": "2025-05-09T13:13:16.169374+00:00", + "fieldMask": [] + }, + "sourceTag": "WEB", + "summary": null, + "url": "https://www.intel.com/content/www/us/en/security-center/advisory/intel-sa-00646.html" + }, + { + "kind": null, + "provenance": { + "source": "osv", + "kind": "reference", + "value": "https://www.kb.cert.org/vuls/id/930724", + "decisionReason": "source-default", + "recordedAt": "2025-05-09T13:13:16.169374+00:00", + "fieldMask": [] + }, + "sourceTag": "WEB", + "summary": null, + "url": "https://www.kb.cert.org/vuls/id/930724" + }, + { + "kind": null, + "provenance": { + "source": "osv", + "kind": "reference", + "value": "https://www.openwall.com/lists/oss-security/2021/12/14/4", + "decisionReason": "source-default", + "recordedAt": "2025-05-09T13:13:16.169374+00:00", + "fieldMask": [] + }, + "sourceTag": "WEB", + "summary": null, + "url": "https://www.openwall.com/lists/oss-security/2021/12/14/4" + }, + { + "kind": null, + "provenance": { + "source": "osv", + "kind": "reference", + "value": "https://www.oracle.com/security-alerts/alert-cve-2021-44228.html", + "decisionReason": "source-default", + "recordedAt": "2025-05-09T13:13:16.169374+00:00", + "fieldMask": [] + }, + "sourceTag": "WEB", + "summary": null, + "url": "https://www.oracle.com/security-alerts/alert-cve-2021-44228.html" + }, + { + "kind": null, + "provenance": { + "source": "osv", + "kind": "reference", + "value": "https://www.oracle.com/security-alerts/cpuapr2022.html", + "decisionReason": "source-default", + "recordedAt": "2025-05-09T13:13:16.169374+00:00", + "fieldMask": [] + }, + "sourceTag": "WEB", + "summary": null, + "url": "https://www.oracle.com/security-alerts/cpuapr2022.html" + }, + { + "kind": null, + "provenance": { + "source": "osv", + "kind": "reference", + "value": "https://www.oracle.com/security-alerts/cpujan2022.html", + "decisionReason": "source-default", + "recordedAt": "2025-05-09T13:13:16.169374+00:00", + "fieldMask": [] + }, + "sourceTag": "WEB", + "summary": null, + "url": "https://www.oracle.com/security-alerts/cpujan2022.html" + }, + { + "kind": null, + "provenance": { + "source": "osv", + "kind": "reference", + "value": "https://www.oracle.com/security-alerts/cpujul2022.html", + "decisionReason": "source-default", + "recordedAt": "2025-05-09T13:13:16.169374+00:00", + "fieldMask": [] + }, + "sourceTag": "WEB", + "summary": null, + "url": "https://www.oracle.com/security-alerts/cpujul2022.html" + } + ], + "severity": "critical", + "summary": "# Impact The fix to address [CVE-2021-44228](https://nvd.nist.gov/vuln/detail/CVE-2021-44228) in Apache Log4j 2.15.0 was incomplete in certain non-default configurations. This could allow attackers with control over Thread Context Map (MDC) input data when the logging configuration uses a non-default Pattern Layout with either a Context Lookup (for example, $${ctx:loginId}) or a Thread Context Map pattern (%X, %mdc, or %MDC) to craft malicious input data using a JNDI Lookup pattern resulting in a remote code execution (RCE) attack. ## Affected packages Only the `org.apache.logging.log4j:log4j-core` package is directly affected by this vulnerability. The `org.apache.logging.log4j:log4j-api` should be kept at the same version as the `org.apache.logging.log4j:log4j-core` package to ensure compatability if in use. # Mitigation Log4j 2.16.0 fixes this issue by removing support for message lookup patterns and disabling JNDI functionality by default. This issue can be mitigated in prior releases (< 2.16.0) by removing the JndiLookup class from the classpath (example: zip -q -d log4j-core-*.jar org/apache/logging/log4j/core/lookup/JndiLookup.class). Log4j 2.15.0 restricts JNDI LDAP lookups to localhost by default. Note that previous mitigations involving configuration such as to set the system property `log4j2.formatMsgNoLookups` to `true` do NOT mitigate this specific vulnerability.", + "title": "Incomplete fix for Apache Log4j vulnerability" + }, + { + "advisoryKey": "GHSA-cjjf-27cc-pvmv", + "affectedPackages": [ + { + "type": "semver", + "identifier": "pkg:pypi/pyload-ng", + "platform": "PyPI", + "versionRanges": [ + { + "fixedVersion": "0.5.0b3.dev91", + "introducedVersion": "0", + "lastAffectedVersion": null, + "primitives": { + "evr": null, + "hasVendorExtensions": false, + "nevra": null, + "semVer": { + "style": "range", + "introduced": "0", + "introducedInclusive": true, + "fixed": "0.5.0b3.dev91", + "fixedInclusive": false, + "lastAffected": null, + "lastAffectedInclusive": true, + "exactValue": null, + "constraintExpression": ">=0 <0.5.0b3.dev91" + }, + "vendorExtensions": null + }, + "provenance": { + "source": "osv", + "kind": "range", + "value": "pkg:pypi/pyload-ng", + "decisionReason": "source-default", + "recordedAt": "2025-10-09T15:59:13.250015+00:00", + "fieldMask": [] + }, + "rangeExpression": ">=0 <0.5.0b3.dev91", + "rangeKind": "semver" + } + ], + "normalizedVersions": [ + { + "scheme": "semver", + "type": "exact", + "min": null, + "minInclusive": null, + "max": null, + "maxInclusive": null, + "value": "0.5.0a5.dev528", + "notes": "osv:PyPI:GHSA-cjjf-27cc-pvmv:pkg:pypi/pyload-ng" + }, + { + "scheme": "semver", + "type": "exact", + "min": null, + "minInclusive": null, + "max": null, + "maxInclusive": null, + "value": "0.5.0a5.dev532", + "notes": "osv:PyPI:GHSA-cjjf-27cc-pvmv:pkg:pypi/pyload-ng" + }, + { + "scheme": "semver", + "type": "exact", + "min": null, + "minInclusive": null, + "max": null, + "maxInclusive": null, + "value": "0.5.0a5.dev535", + "notes": "osv:PyPI:GHSA-cjjf-27cc-pvmv:pkg:pypi/pyload-ng" + }, + { + "scheme": "semver", + "type": "exact", + "min": null, + "minInclusive": null, + "max": null, + "maxInclusive": null, + "value": "0.5.0a5.dev536", + "notes": "osv:PyPI:GHSA-cjjf-27cc-pvmv:pkg:pypi/pyload-ng" + }, + { + "scheme": "semver", + "type": "exact", + "min": null, + "minInclusive": null, + "max": null, + "maxInclusive": null, + "value": "0.5.0a5.dev537", + "notes": "osv:PyPI:GHSA-cjjf-27cc-pvmv:pkg:pypi/pyload-ng" + }, + { + "scheme": "semver", + "type": "exact", + "min": null, + "minInclusive": null, + "max": null, + "maxInclusive": null, + "value": "0.5.0a5.dev539", + "notes": "osv:PyPI:GHSA-cjjf-27cc-pvmv:pkg:pypi/pyload-ng" + }, + { + "scheme": "semver", + "type": "exact", + "min": null, + "minInclusive": null, + "max": null, + "maxInclusive": null, + "value": "0.5.0a5.dev540", + "notes": "osv:PyPI:GHSA-cjjf-27cc-pvmv:pkg:pypi/pyload-ng" + }, + { + "scheme": "semver", + "type": "exact", + "min": null, + "minInclusive": null, + "max": null, + "maxInclusive": null, + "value": "0.5.0a5.dev545", + "notes": "osv:PyPI:GHSA-cjjf-27cc-pvmv:pkg:pypi/pyload-ng" + }, + { + "scheme": "semver", + "type": "exact", + "min": null, + "minInclusive": null, + "max": null, + "maxInclusive": null, + "value": "0.5.0a5.dev562", + "notes": "osv:PyPI:GHSA-cjjf-27cc-pvmv:pkg:pypi/pyload-ng" + }, + { + "scheme": "semver", + "type": "exact", + "min": null, + "minInclusive": null, + "max": null, + "maxInclusive": null, + "value": "0.5.0a5.dev564", + "notes": "osv:PyPI:GHSA-cjjf-27cc-pvmv:pkg:pypi/pyload-ng" + }, + { + "scheme": "semver", + "type": "exact", + "min": null, + "minInclusive": null, + "max": null, + "maxInclusive": null, + "value": "0.5.0a5.dev565", + "notes": "osv:PyPI:GHSA-cjjf-27cc-pvmv:pkg:pypi/pyload-ng" + }, + { + "scheme": "semver", + "type": "exact", + "min": null, + "minInclusive": null, + "max": null, + "maxInclusive": null, + "value": "0.5.0a6.dev570", + "notes": "osv:PyPI:GHSA-cjjf-27cc-pvmv:pkg:pypi/pyload-ng" + }, + { + "scheme": "semver", + "type": "exact", + "min": null, + "minInclusive": null, + "max": null, + "maxInclusive": null, + "value": "0.5.0a6.dev578", + "notes": "osv:PyPI:GHSA-cjjf-27cc-pvmv:pkg:pypi/pyload-ng" + }, + { + "scheme": "semver", + "type": "exact", + "min": null, + "minInclusive": null, + "max": null, + "maxInclusive": null, + "value": "0.5.0a6.dev587", + "notes": "osv:PyPI:GHSA-cjjf-27cc-pvmv:pkg:pypi/pyload-ng" + }, + { + "scheme": "semver", + "type": "exact", + "min": null, + "minInclusive": null, + "max": null, + "maxInclusive": null, + "value": "0.5.0a7.dev596", + "notes": "osv:PyPI:GHSA-cjjf-27cc-pvmv:pkg:pypi/pyload-ng" + }, + { + "scheme": "semver", + "type": "exact", + "min": null, + "minInclusive": null, + "max": null, + "maxInclusive": null, + "value": "0.5.0a8.dev602", + "notes": "osv:PyPI:GHSA-cjjf-27cc-pvmv:pkg:pypi/pyload-ng" + }, + { + "scheme": "semver", + "type": "exact", + "min": null, + "minInclusive": null, + "max": null, + "maxInclusive": null, + "value": "0.5.0a9.dev615", + "notes": "osv:PyPI:GHSA-cjjf-27cc-pvmv:pkg:pypi/pyload-ng" + }, + { + "scheme": "semver", + "type": "exact", + "min": null, + "minInclusive": null, + "max": null, + "maxInclusive": null, + "value": "0.5.0a9.dev629", + "notes": "osv:PyPI:GHSA-cjjf-27cc-pvmv:pkg:pypi/pyload-ng" + }, + { + "scheme": "semver", + "type": "exact", + "min": null, + "minInclusive": null, + "max": null, + "maxInclusive": null, + "value": "0.5.0a9.dev632", + "notes": "osv:PyPI:GHSA-cjjf-27cc-pvmv:pkg:pypi/pyload-ng" + }, + { + "scheme": "semver", + "type": "exact", + "min": null, + "minInclusive": null, + "max": null, + "maxInclusive": null, + "value": "0.5.0a9.dev641", + "notes": "osv:PyPI:GHSA-cjjf-27cc-pvmv:pkg:pypi/pyload-ng" + }, + { + "scheme": "semver", + "type": "exact", + "min": null, + "minInclusive": null, + "max": null, + "maxInclusive": null, + "value": "0.5.0a9.dev643", + "notes": "osv:PyPI:GHSA-cjjf-27cc-pvmv:pkg:pypi/pyload-ng" + }, + { + "scheme": "semver", + "type": "exact", + "min": null, + "minInclusive": null, + "max": null, + "maxInclusive": null, + "value": "0.5.0a9.dev655", + "notes": "osv:PyPI:GHSA-cjjf-27cc-pvmv:pkg:pypi/pyload-ng" + }, + { + "scheme": "semver", + "type": "exact", + "min": null, + "minInclusive": null, + "max": null, + "maxInclusive": null, + "value": "0.5.0a9.dev806", + "notes": "osv:PyPI:GHSA-cjjf-27cc-pvmv:pkg:pypi/pyload-ng" + }, + { + "scheme": "semver", + "type": "exact", + "min": null, + "minInclusive": null, + "max": null, + "maxInclusive": null, + "value": "0.5.0b1.dev1", + "notes": "osv:PyPI:GHSA-cjjf-27cc-pvmv:pkg:pypi/pyload-ng" + }, + { + "scheme": "semver", + "type": "exact", + "min": null, + "minInclusive": null, + "max": null, + "maxInclusive": null, + "value": "0.5.0b1.dev2", + "notes": "osv:PyPI:GHSA-cjjf-27cc-pvmv:pkg:pypi/pyload-ng" + }, + { + "scheme": "semver", + "type": "exact", + "min": null, + "minInclusive": null, + "max": null, + "maxInclusive": null, + "value": "0.5.0b1.dev3", + "notes": "osv:PyPI:GHSA-cjjf-27cc-pvmv:pkg:pypi/pyload-ng" + }, + { + "scheme": "semver", + "type": "exact", + "min": null, + "minInclusive": null, + "max": null, + "maxInclusive": null, + "value": "0.5.0b1.dev4", + "notes": "osv:PyPI:GHSA-cjjf-27cc-pvmv:pkg:pypi/pyload-ng" + }, + { + "scheme": "semver", + "type": "exact", + "min": null, + "minInclusive": null, + "max": null, + "maxInclusive": null, + "value": "0.5.0b1.dev5", + "notes": "osv:PyPI:GHSA-cjjf-27cc-pvmv:pkg:pypi/pyload-ng" + }, + { + "scheme": "semver", + "type": "exact", + "min": null, + "minInclusive": null, + "max": null, + "maxInclusive": null, + "value": "0.5.0b2.dev10", + "notes": "osv:PyPI:GHSA-cjjf-27cc-pvmv:pkg:pypi/pyload-ng" + }, + { + "scheme": "semver", + "type": "exact", + "min": null, + "minInclusive": null, + "max": null, + "maxInclusive": null, + "value": "0.5.0b2.dev11", + "notes": "osv:PyPI:GHSA-cjjf-27cc-pvmv:pkg:pypi/pyload-ng" + }, + { + "scheme": "semver", + "type": "exact", + "min": null, + "minInclusive": null, + "max": null, + "maxInclusive": null, + "value": "0.5.0b2.dev12", + "notes": "osv:PyPI:GHSA-cjjf-27cc-pvmv:pkg:pypi/pyload-ng" + }, + { + "scheme": "semver", + "type": "exact", + "min": null, + "minInclusive": null, + "max": null, + "maxInclusive": null, + "value": "0.5.0b2.dev9", + "notes": "osv:PyPI:GHSA-cjjf-27cc-pvmv:pkg:pypi/pyload-ng" + }, + { + "scheme": "semver", + "type": "exact", + "min": null, + "minInclusive": null, + "max": null, + "maxInclusive": null, + "value": "0.5.0b3.dev13", + "notes": "osv:PyPI:GHSA-cjjf-27cc-pvmv:pkg:pypi/pyload-ng" + }, + { + "scheme": "semver", + "type": "exact", + "min": null, + "minInclusive": null, + "max": null, + "maxInclusive": null, + "value": "0.5.0b3.dev14", + "notes": "osv:PyPI:GHSA-cjjf-27cc-pvmv:pkg:pypi/pyload-ng" + }, + { + "scheme": "semver", + "type": "exact", + "min": null, + "minInclusive": null, + "max": null, + "maxInclusive": null, + "value": "0.5.0b3.dev17", + "notes": "osv:PyPI:GHSA-cjjf-27cc-pvmv:pkg:pypi/pyload-ng" + }, + { + "scheme": "semver", + "type": "exact", + "min": null, + "minInclusive": null, + "max": null, + "maxInclusive": null, + "value": "0.5.0b3.dev18", + "notes": "osv:PyPI:GHSA-cjjf-27cc-pvmv:pkg:pypi/pyload-ng" + }, + { + "scheme": "semver", + "type": "exact", + "min": null, + "minInclusive": null, + "max": null, + "maxInclusive": null, + "value": "0.5.0b3.dev19", + "notes": "osv:PyPI:GHSA-cjjf-27cc-pvmv:pkg:pypi/pyload-ng" + }, + { + "scheme": "semver", + "type": "exact", + "min": null, + "minInclusive": null, + "max": null, + "maxInclusive": null, + "value": "0.5.0b3.dev20", + "notes": "osv:PyPI:GHSA-cjjf-27cc-pvmv:pkg:pypi/pyload-ng" + }, + { + "scheme": "semver", + "type": "exact", + "min": null, + "minInclusive": null, + "max": null, + "maxInclusive": null, + "value": "0.5.0b3.dev21", + "notes": "osv:PyPI:GHSA-cjjf-27cc-pvmv:pkg:pypi/pyload-ng" + }, + { + "scheme": "semver", + "type": "exact", + "min": null, + "minInclusive": null, + "max": null, + "maxInclusive": null, + "value": "0.5.0b3.dev22", + "notes": "osv:PyPI:GHSA-cjjf-27cc-pvmv:pkg:pypi/pyload-ng" + }, + { + "scheme": "semver", + "type": "exact", + "min": null, + "minInclusive": null, + "max": null, + "maxInclusive": null, + "value": "0.5.0b3.dev24", + "notes": "osv:PyPI:GHSA-cjjf-27cc-pvmv:pkg:pypi/pyload-ng" + }, + { + "scheme": "semver", + "type": "exact", + "min": null, + "minInclusive": null, + "max": null, + "maxInclusive": null, + "value": "0.5.0b3.dev26", + "notes": "osv:PyPI:GHSA-cjjf-27cc-pvmv:pkg:pypi/pyload-ng" + }, + { + "scheme": "semver", + "type": "exact", + "min": null, + "minInclusive": null, + "max": null, + "maxInclusive": null, + "value": "0.5.0b3.dev27", + "notes": "osv:PyPI:GHSA-cjjf-27cc-pvmv:pkg:pypi/pyload-ng" + }, + { + "scheme": "semver", + "type": "exact", + "min": null, + "minInclusive": null, + "max": null, + "maxInclusive": null, + "value": "0.5.0b3.dev28", + "notes": "osv:PyPI:GHSA-cjjf-27cc-pvmv:pkg:pypi/pyload-ng" + }, + { + "scheme": "semver", + "type": "exact", + "min": null, + "minInclusive": null, + "max": null, + "maxInclusive": null, + "value": "0.5.0b3.dev29", + "notes": "osv:PyPI:GHSA-cjjf-27cc-pvmv:pkg:pypi/pyload-ng" + }, + { + "scheme": "semver", + "type": "exact", + "min": null, + "minInclusive": null, + "max": null, + "maxInclusive": null, + "value": "0.5.0b3.dev30", + "notes": "osv:PyPI:GHSA-cjjf-27cc-pvmv:pkg:pypi/pyload-ng" + }, + { + "scheme": "semver", + "type": "exact", + "min": null, + "minInclusive": null, + "max": null, + "maxInclusive": null, + "value": "0.5.0b3.dev31", + "notes": "osv:PyPI:GHSA-cjjf-27cc-pvmv:pkg:pypi/pyload-ng" + }, + { + "scheme": "semver", + "type": "exact", + "min": null, + "minInclusive": null, + "max": null, + "maxInclusive": null, + "value": "0.5.0b3.dev32", + "notes": "osv:PyPI:GHSA-cjjf-27cc-pvmv:pkg:pypi/pyload-ng" + }, + { + "scheme": "semver", + "type": "exact", + "min": null, + "minInclusive": null, + "max": null, + "maxInclusive": null, + "value": "0.5.0b3.dev33", + "notes": "osv:PyPI:GHSA-cjjf-27cc-pvmv:pkg:pypi/pyload-ng" + }, + { + "scheme": "semver", + "type": "exact", + "min": null, + "minInclusive": null, + "max": null, + "maxInclusive": null, + "value": "0.5.0b3.dev34", + "notes": "osv:PyPI:GHSA-cjjf-27cc-pvmv:pkg:pypi/pyload-ng" + }, + { + "scheme": "semver", + "type": "exact", + "min": null, + "minInclusive": null, + "max": null, + "maxInclusive": null, + "value": "0.5.0b3.dev35", + "notes": "osv:PyPI:GHSA-cjjf-27cc-pvmv:pkg:pypi/pyload-ng" + }, + { + "scheme": "semver", + "type": "exact", + "min": null, + "minInclusive": null, + "max": null, + "maxInclusive": null, + "value": "0.5.0b3.dev38", + "notes": "osv:PyPI:GHSA-cjjf-27cc-pvmv:pkg:pypi/pyload-ng" + }, + { + "scheme": "semver", + "type": "exact", + "min": null, + "minInclusive": null, + "max": null, + "maxInclusive": null, + "value": "0.5.0b3.dev39", + "notes": "osv:PyPI:GHSA-cjjf-27cc-pvmv:pkg:pypi/pyload-ng" + }, + { + "scheme": "semver", + "type": "exact", + "min": null, + "minInclusive": null, + "max": null, + "maxInclusive": null, + "value": "0.5.0b3.dev40", + "notes": "osv:PyPI:GHSA-cjjf-27cc-pvmv:pkg:pypi/pyload-ng" + }, + { + "scheme": "semver", + "type": "exact", + "min": null, + "minInclusive": null, + "max": null, + "maxInclusive": null, + "value": "0.5.0b3.dev41", + "notes": "osv:PyPI:GHSA-cjjf-27cc-pvmv:pkg:pypi/pyload-ng" + }, + { + "scheme": "semver", + "type": "exact", + "min": null, + "minInclusive": null, + "max": null, + "maxInclusive": null, + "value": "0.5.0b3.dev42", + "notes": "osv:PyPI:GHSA-cjjf-27cc-pvmv:pkg:pypi/pyload-ng" + }, + { + "scheme": "semver", + "type": "exact", + "min": null, + "minInclusive": null, + "max": null, + "maxInclusive": null, + "value": "0.5.0b3.dev43", + "notes": "osv:PyPI:GHSA-cjjf-27cc-pvmv:pkg:pypi/pyload-ng" + }, + { + "scheme": "semver", + "type": "exact", + "min": null, + "minInclusive": null, + "max": null, + "maxInclusive": null, + "value": "0.5.0b3.dev44", + "notes": "osv:PyPI:GHSA-cjjf-27cc-pvmv:pkg:pypi/pyload-ng" + }, + { + "scheme": "semver", + "type": "exact", + "min": null, + "minInclusive": null, + "max": null, + "maxInclusive": null, + "value": "0.5.0b3.dev45", + "notes": "osv:PyPI:GHSA-cjjf-27cc-pvmv:pkg:pypi/pyload-ng" + }, + { + "scheme": "semver", + "type": "exact", + "min": null, + "minInclusive": null, + "max": null, + "maxInclusive": null, + "value": "0.5.0b3.dev46", + "notes": "osv:PyPI:GHSA-cjjf-27cc-pvmv:pkg:pypi/pyload-ng" + }, + { + "scheme": "semver", + "type": "exact", + "min": null, + "minInclusive": null, + "max": null, + "maxInclusive": null, + "value": "0.5.0b3.dev47", + "notes": "osv:PyPI:GHSA-cjjf-27cc-pvmv:pkg:pypi/pyload-ng" + }, + { + "scheme": "semver", + "type": "exact", + "min": null, + "minInclusive": null, + "max": null, + "maxInclusive": null, + "value": "0.5.0b3.dev48", + "notes": "osv:PyPI:GHSA-cjjf-27cc-pvmv:pkg:pypi/pyload-ng" + }, + { + "scheme": "semver", + "type": "exact", + "min": null, + "minInclusive": null, + "max": null, + "maxInclusive": null, + "value": "0.5.0b3.dev49", + "notes": "osv:PyPI:GHSA-cjjf-27cc-pvmv:pkg:pypi/pyload-ng" + }, + { + "scheme": "semver", + "type": "exact", + "min": null, + "minInclusive": null, + "max": null, + "maxInclusive": null, + "value": "0.5.0b3.dev50", + "notes": "osv:PyPI:GHSA-cjjf-27cc-pvmv:pkg:pypi/pyload-ng" + }, + { + "scheme": "semver", + "type": "exact", + "min": null, + "minInclusive": null, + "max": null, + "maxInclusive": null, + "value": "0.5.0b3.dev51", + "notes": "osv:PyPI:GHSA-cjjf-27cc-pvmv:pkg:pypi/pyload-ng" + }, + { + "scheme": "semver", + "type": "exact", + "min": null, + "minInclusive": null, + "max": null, + "maxInclusive": null, + "value": "0.5.0b3.dev52", + "notes": "osv:PyPI:GHSA-cjjf-27cc-pvmv:pkg:pypi/pyload-ng" + }, + { + "scheme": "semver", + "type": "exact", + "min": null, + "minInclusive": null, + "max": null, + "maxInclusive": null, + "value": "0.5.0b3.dev53", + "notes": "osv:PyPI:GHSA-cjjf-27cc-pvmv:pkg:pypi/pyload-ng" + }, + { + "scheme": "semver", + "type": "exact", + "min": null, + "minInclusive": null, + "max": null, + "maxInclusive": null, + "value": "0.5.0b3.dev54", + "notes": "osv:PyPI:GHSA-cjjf-27cc-pvmv:pkg:pypi/pyload-ng" + }, + { + "scheme": "semver", + "type": "exact", + "min": null, + "minInclusive": null, + "max": null, + "maxInclusive": null, + "value": "0.5.0b3.dev57", + "notes": "osv:PyPI:GHSA-cjjf-27cc-pvmv:pkg:pypi/pyload-ng" + }, + { + "scheme": "semver", + "type": "exact", + "min": null, + "minInclusive": null, + "max": null, + "maxInclusive": null, + "value": "0.5.0b3.dev60", + "notes": "osv:PyPI:GHSA-cjjf-27cc-pvmv:pkg:pypi/pyload-ng" + }, + { + "scheme": "semver", + "type": "exact", + "min": null, + "minInclusive": null, + "max": null, + "maxInclusive": null, + "value": "0.5.0b3.dev62", + "notes": "osv:PyPI:GHSA-cjjf-27cc-pvmv:pkg:pypi/pyload-ng" + }, + { + "scheme": "semver", + "type": "exact", + "min": null, + "minInclusive": null, + "max": null, + "maxInclusive": null, + "value": "0.5.0b3.dev64", + "notes": "osv:PyPI:GHSA-cjjf-27cc-pvmv:pkg:pypi/pyload-ng" + }, + { + "scheme": "semver", + "type": "exact", + "min": null, + "minInclusive": null, + "max": null, + "maxInclusive": null, + "value": "0.5.0b3.dev65", + "notes": "osv:PyPI:GHSA-cjjf-27cc-pvmv:pkg:pypi/pyload-ng" + }, + { + "scheme": "semver", + "type": "exact", + "min": null, + "minInclusive": null, + "max": null, + "maxInclusive": null, + "value": "0.5.0b3.dev66", + "notes": "osv:PyPI:GHSA-cjjf-27cc-pvmv:pkg:pypi/pyload-ng" + }, + { + "scheme": "semver", + "type": "exact", + "min": null, + "minInclusive": null, + "max": null, + "maxInclusive": null, + "value": "0.5.0b3.dev67", + "notes": "osv:PyPI:GHSA-cjjf-27cc-pvmv:pkg:pypi/pyload-ng" + }, + { + "scheme": "semver", + "type": "exact", + "min": null, + "minInclusive": null, + "max": null, + "maxInclusive": null, + "value": "0.5.0b3.dev68", + "notes": "osv:PyPI:GHSA-cjjf-27cc-pvmv:pkg:pypi/pyload-ng" + }, + { + "scheme": "semver", + "type": "exact", + "min": null, + "minInclusive": null, + "max": null, + "maxInclusive": null, + "value": "0.5.0b3.dev69", + "notes": "osv:PyPI:GHSA-cjjf-27cc-pvmv:pkg:pypi/pyload-ng" + }, + { + "scheme": "semver", + "type": "exact", + "min": null, + "minInclusive": null, + "max": null, + "maxInclusive": null, + "value": "0.5.0b3.dev70", + "notes": "osv:PyPI:GHSA-cjjf-27cc-pvmv:pkg:pypi/pyload-ng" + }, + { + "scheme": "semver", + "type": "exact", + "min": null, + "minInclusive": null, + "max": null, + "maxInclusive": null, + "value": "0.5.0b3.dev71", + "notes": "osv:PyPI:GHSA-cjjf-27cc-pvmv:pkg:pypi/pyload-ng" + }, + { + "scheme": "semver", + "type": "exact", + "min": null, + "minInclusive": null, + "max": null, + "maxInclusive": null, + "value": "0.5.0b3.dev72", + "notes": "osv:PyPI:GHSA-cjjf-27cc-pvmv:pkg:pypi/pyload-ng" + }, + { + "scheme": "semver", + "type": "exact", + "min": null, + "minInclusive": null, + "max": null, + "maxInclusive": null, + "value": "0.5.0b3.dev73", + "notes": "osv:PyPI:GHSA-cjjf-27cc-pvmv:pkg:pypi/pyload-ng" + }, + { + "scheme": "semver", + "type": "exact", + "min": null, + "minInclusive": null, + "max": null, + "maxInclusive": null, + "value": "0.5.0b3.dev74", + "notes": "osv:PyPI:GHSA-cjjf-27cc-pvmv:pkg:pypi/pyload-ng" + }, + { + "scheme": "semver", + "type": "exact", + "min": null, + "minInclusive": null, + "max": null, + "maxInclusive": null, + "value": "0.5.0b3.dev75", + "notes": "osv:PyPI:GHSA-cjjf-27cc-pvmv:pkg:pypi/pyload-ng" + }, + { + "scheme": "semver", + "type": "exact", + "min": null, + "minInclusive": null, + "max": null, + "maxInclusive": null, + "value": "0.5.0b3.dev76", + "notes": "osv:PyPI:GHSA-cjjf-27cc-pvmv:pkg:pypi/pyload-ng" + }, + { + "scheme": "semver", + "type": "exact", + "min": null, + "minInclusive": null, + "max": null, + "maxInclusive": null, + "value": "0.5.0b3.dev77", + "notes": "osv:PyPI:GHSA-cjjf-27cc-pvmv:pkg:pypi/pyload-ng" + }, + { + "scheme": "semver", + "type": "exact", + "min": null, + "minInclusive": null, + "max": null, + "maxInclusive": null, + "value": "0.5.0b3.dev78", + "notes": "osv:PyPI:GHSA-cjjf-27cc-pvmv:pkg:pypi/pyload-ng" + }, + { + "scheme": "semver", + "type": "exact", + "min": null, + "minInclusive": null, + "max": null, + "maxInclusive": null, + "value": "0.5.0b3.dev79", + "notes": "osv:PyPI:GHSA-cjjf-27cc-pvmv:pkg:pypi/pyload-ng" + }, + { + "scheme": "semver", + "type": "exact", + "min": null, + "minInclusive": null, + "max": null, + "maxInclusive": null, + "value": "0.5.0b3.dev80", + "notes": "osv:PyPI:GHSA-cjjf-27cc-pvmv:pkg:pypi/pyload-ng" + }, + { + "scheme": "semver", + "type": "exact", + "min": null, + "minInclusive": null, + "max": null, + "maxInclusive": null, + "value": "0.5.0b3.dev81", + "notes": "osv:PyPI:GHSA-cjjf-27cc-pvmv:pkg:pypi/pyload-ng" + }, + { + "scheme": "semver", + "type": "exact", + "min": null, + "minInclusive": null, + "max": null, + "maxInclusive": null, + "value": "0.5.0b3.dev82", + "notes": "osv:PyPI:GHSA-cjjf-27cc-pvmv:pkg:pypi/pyload-ng" + }, + { + "scheme": "semver", + "type": "exact", + "min": null, + "minInclusive": null, + "max": null, + "maxInclusive": null, + "value": "0.5.0b3.dev85", + "notes": "osv:PyPI:GHSA-cjjf-27cc-pvmv:pkg:pypi/pyload-ng" + }, + { + "scheme": "semver", + "type": "exact", + "min": null, + "minInclusive": null, + "max": null, + "maxInclusive": null, + "value": "0.5.0b3.dev87", + "notes": "osv:PyPI:GHSA-cjjf-27cc-pvmv:pkg:pypi/pyload-ng" + }, + { + "scheme": "semver", + "type": "exact", + "min": null, + "minInclusive": null, + "max": null, + "maxInclusive": null, + "value": "0.5.0b3.dev88", + "notes": "osv:PyPI:GHSA-cjjf-27cc-pvmv:pkg:pypi/pyload-ng" + }, + { + "scheme": "semver", + "type": "exact", + "min": null, + "minInclusive": null, + "max": null, + "maxInclusive": null, + "value": "0.5.0b3.dev89", + "notes": "osv:PyPI:GHSA-cjjf-27cc-pvmv:pkg:pypi/pyload-ng" + }, + { + "scheme": "semver", + "type": "exact", + "min": null, + "minInclusive": null, + "max": null, + "maxInclusive": null, + "value": "0.5.0b3.dev90", + "notes": "osv:PyPI:GHSA-cjjf-27cc-pvmv:pkg:pypi/pyload-ng" + }, + { + "scheme": "semver", + "type": "range", + "min": "0", + "minInclusive": true, + "max": "0.5.0b3.dev91", + "maxInclusive": false, + "value": null, + "notes": "osv:PyPI:GHSA-cjjf-27cc-pvmv:pkg:pypi/pyload-ng" + } + ], + "statuses": [], + "provenance": [ + { + "source": "osv", + "kind": "affected", + "value": "pkg:pypi/pyload-ng", + "decisionReason": "source-default", + "recordedAt": "2025-10-09T15:59:13.250015+00:00", + "fieldMask": [] + } + ] + } + ], + "aliases": [ + "CVE-2025-61773", + "GHSA-cjjf-27cc-pvmv" + ], + "cvssMetrics": [ + { + "baseScore": 8.1, + "baseSeverity": "high", + "provenance": { + "source": "osv", + "kind": "cvss", + "value": "CVSS_V3", + "decisionReason": "source-default", + "recordedAt": "2025-10-09T15:59:13.250015+00:00", + "fieldMask": [] + }, + "vector": "CVSS:3.1/AV:N/AC:L/PR:N/UI:R/S:U/C:H/I:H/A:N", + "version": "3.1" + } + ], + "exploitKnown": false, + "language": "en", + "modified": "2025-10-09T15:59:13.250015+00:00", + "provenance": [ + { + "source": "osv", + "kind": "document", + "value": "https://osv.dev/vulnerability/GHSA-cjjf-27cc-pvmv", + "decisionReason": "source-default", + "recordedAt": "2025-10-09T15:19:48+00:00", + "fieldMask": [] + }, + { + "source": "osv", + "kind": "mapping", + "value": "GHSA-cjjf-27cc-pvmv", + "decisionReason": "source-default", + "recordedAt": "2025-10-09T15:59:13.250015+00:00", + "fieldMask": [] + } + ], + "published": "2025-10-09T15:19:48+00:00", + "references": [ + { + "kind": null, + "provenance": { + "source": "osv", + "kind": "reference", + "value": "https://github.com/pyload/pyload", + "decisionReason": "source-default", + "recordedAt": "2025-10-09T15:59:13.250015+00:00", + "fieldMask": [] + }, + "sourceTag": "PACKAGE", + "summary": null, + "url": "https://github.com/pyload/pyload" + }, + { + "kind": null, + "provenance": { + "source": "osv", + "kind": "reference", + "value": "https://github.com/pyload/pyload/commit/5823327d0b797161c7195a1f660266d30a69f0ca", + "decisionReason": "source-default", + "recordedAt": "2025-10-09T15:59:13.250015+00:00", + "fieldMask": [] + }, + "sourceTag": "WEB", + "summary": null, + "url": "https://github.com/pyload/pyload/commit/5823327d0b797161c7195a1f660266d30a69f0ca" + }, + { + "kind": null, + "provenance": { + "source": "osv", + "kind": "reference", + "value": "https://github.com/pyload/pyload/pull/4624", + "decisionReason": "source-default", + "recordedAt": "2025-10-09T15:59:13.250015+00:00", + "fieldMask": [] + }, + "sourceTag": "WEB", + "summary": null, + "url": "https://github.com/pyload/pyload/pull/4624" + }, + { + "kind": null, + "provenance": { + "source": "osv", + "kind": "reference", + "value": "https://github.com/pyload/pyload/security/advisories/GHSA-cjjf-27cc-pvmv", + "decisionReason": "source-default", + "recordedAt": "2025-10-09T15:59:13.250015+00:00", + "fieldMask": [] + }, + "sourceTag": "WEB", + "summary": null, + "url": "https://github.com/pyload/pyload/security/advisories/GHSA-cjjf-27cc-pvmv" + } + ], + "severity": "high", + "summary": "### Summary pyLoad web interface contained insufficient input validation in both the Captcha script endpoint and the Click'N'Load (CNL) Blueprint. This flaw allowed untrusted user input to be processed unsafely, which could be exploited by an attacker to inject arbitrary content into the web UI or manipulate request handling. The vulnerability could lead to client-side code execution (XSS) or other unintended behaviors when a malicious payload is submitted. user-supplied parameters from HTTP requests were not adequately validated or sanitized before being passed into the application logic and response generation. This allowed crafted input to alter the expected execution flow. CNL (Click'N'Load) blueprint exposed unsafe handling of untrusted parameters in HTTP requests. The application did not consistently enforce input validation or encoding, making it possible for an attacker to craft malicious requests. ### PoC 1. Run a vulnerable version of pyLoad prior to commit [`f9d27f2`](https://github.com/pyload/pyload/pull/4624). 2. Start the web UI and access the Captcha or CNL endpoints. 3. Submit a crafted request containing malicious JavaScript payloads in unvalidated parameters (`/flash/addcrypted2?jk=function(){alert(1)}&crypted=12345`). 4. Observe that the payload is reflected and executed in the client’s browser, demonstrating cross-site scripting (XSS). Example request: ```http GET /flash/addcrypted2?jk=function(){alert(1)}&crypted=12345 HTTP/1.1 Host: 127.0.0.1:8000 Content-Type: application/x-www-form-urlencoded Content-Length: 107 ``` ### Impact Exploiting this vulnerability allows an attacker to inject and execute arbitrary JavaScript within the browser session of a user accessing the pyLoad Web UI. In practice, this means an attacker could impersonate an administrator, steal authentication cookies or tokens, and perform unauthorized actions on behalf of the victim. Because the affected endpoints are part of the core interface, a successful attack undermines the trust and security of the entire application, potentially leading to a full compromise of the management interface and the data it controls. The impact is particularly severe in cases where the Web UI is exposed over a network without additional access restrictions, as it enables remote attackers to directly target users with crafted links or requests that trigger the vulnerability.", + "title": "pyLoad CNL and captcha handlers allow Code Injection via unsanitized parameters" + }, + { + "advisoryKey": "GHSA-wv4w-6qv2-qqfg", + "affectedPackages": [ + { + "type": "semver", + "identifier": "pkg:pypi/social-auth-app-django", + "platform": "PyPI", + "versionRanges": [ + { + "fixedVersion": "5.6.0", + "introducedVersion": "0", + "lastAffectedVersion": null, + "primitives": { + "evr": null, + "hasVendorExtensions": false, + "nevra": null, + "semVer": { + "style": "range", + "introduced": "0", + "introducedInclusive": true, + "fixed": "5.6.0", + "fixedInclusive": false, + "lastAffected": null, + "lastAffectedInclusive": true, + "exactValue": null, + "constraintExpression": ">=0 <5.6.0" + }, + "vendorExtensions": null + }, + "provenance": { + "source": "osv", + "kind": "range", + "value": "pkg:pypi/social-auth-app-django", + "decisionReason": "source-default", + "recordedAt": "2025-10-09T17:57:29.916841+00:00", + "fieldMask": [] + }, + "rangeExpression": ">=0 <5.6.0", + "rangeKind": "semver" + } + ], + "normalizedVersions": [ + { + "scheme": "semver", + "type": "exact", + "min": null, + "minInclusive": null, + "max": null, + "maxInclusive": null, + "value": "0.0.1", + "notes": "osv:PyPI:GHSA-wv4w-6qv2-qqfg:pkg:pypi/social-auth-app-django" + }, + { + "scheme": "semver", + "type": "exact", + "min": null, + "minInclusive": null, + "max": null, + "maxInclusive": null, + "value": "0.1.0", + "notes": "osv:PyPI:GHSA-wv4w-6qv2-qqfg:pkg:pypi/social-auth-app-django" + }, + { + "scheme": "semver", + "type": "exact", + "min": null, + "minInclusive": null, + "max": null, + "maxInclusive": null, + "value": "1.0.0", + "notes": "osv:PyPI:GHSA-wv4w-6qv2-qqfg:pkg:pypi/social-auth-app-django" + }, + { + "scheme": "semver", + "type": "exact", + "min": null, + "minInclusive": null, + "max": null, + "maxInclusive": null, + "value": "1.0.1", + "notes": "osv:PyPI:GHSA-wv4w-6qv2-qqfg:pkg:pypi/social-auth-app-django" + }, + { + "scheme": "semver", + "type": "exact", + "min": null, + "minInclusive": null, + "max": null, + "maxInclusive": null, + "value": "1.1.0", + "notes": "osv:PyPI:GHSA-wv4w-6qv2-qqfg:pkg:pypi/social-auth-app-django" + }, + { + "scheme": "semver", + "type": "exact", + "min": null, + "minInclusive": null, + "max": null, + "maxInclusive": null, + "value": "1.2.0", + "notes": "osv:PyPI:GHSA-wv4w-6qv2-qqfg:pkg:pypi/social-auth-app-django" + }, + { + "scheme": "semver", + "type": "exact", + "min": null, + "minInclusive": null, + "max": null, + "maxInclusive": null, + "value": "2.0.0", + "notes": "osv:PyPI:GHSA-wv4w-6qv2-qqfg:pkg:pypi/social-auth-app-django" + }, + { + "scheme": "semver", + "type": "exact", + "min": null, + "minInclusive": null, + "max": null, + "maxInclusive": null, + "value": "2.1.0", + "notes": "osv:PyPI:GHSA-wv4w-6qv2-qqfg:pkg:pypi/social-auth-app-django" + }, + { + "scheme": "semver", + "type": "exact", + "min": null, + "minInclusive": null, + "max": null, + "maxInclusive": null, + "value": "3.0.0", + "notes": "osv:PyPI:GHSA-wv4w-6qv2-qqfg:pkg:pypi/social-auth-app-django" + }, + { + "scheme": "semver", + "type": "exact", + "min": null, + "minInclusive": null, + "max": null, + "maxInclusive": null, + "value": "3.1.0", + "notes": "osv:PyPI:GHSA-wv4w-6qv2-qqfg:pkg:pypi/social-auth-app-django" + }, + { + "scheme": "semver", + "type": "exact", + "min": null, + "minInclusive": null, + "max": null, + "maxInclusive": null, + "value": "3.3.0", + "notes": "osv:PyPI:GHSA-wv4w-6qv2-qqfg:pkg:pypi/social-auth-app-django" + }, + { + "scheme": "semver", + "type": "exact", + "min": null, + "minInclusive": null, + "max": null, + "maxInclusive": null, + "value": "3.4.0", + "notes": "osv:PyPI:GHSA-wv4w-6qv2-qqfg:pkg:pypi/social-auth-app-django" + }, + { + "scheme": "semver", + "type": "exact", + "min": null, + "minInclusive": null, + "max": null, + "maxInclusive": null, + "value": "4.0.0", + "notes": "osv:PyPI:GHSA-wv4w-6qv2-qqfg:pkg:pypi/social-auth-app-django" + }, + { + "scheme": "semver", + "type": "exact", + "min": null, + "minInclusive": null, + "max": null, + "maxInclusive": null, + "value": "5.0.0", + "notes": "osv:PyPI:GHSA-wv4w-6qv2-qqfg:pkg:pypi/social-auth-app-django" + }, + { + "scheme": "semver", + "type": "exact", + "min": null, + "minInclusive": null, + "max": null, + "maxInclusive": null, + "value": "5.1.0", + "notes": "osv:PyPI:GHSA-wv4w-6qv2-qqfg:pkg:pypi/social-auth-app-django" + }, + { + "scheme": "semver", + "type": "exact", + "min": null, + "minInclusive": null, + "max": null, + "maxInclusive": null, + "value": "5.2.0", + "notes": "osv:PyPI:GHSA-wv4w-6qv2-qqfg:pkg:pypi/social-auth-app-django" + }, + { + "scheme": "semver", + "type": "exact", + "min": null, + "minInclusive": null, + "max": null, + "maxInclusive": null, + "value": "5.3.0", + "notes": "osv:PyPI:GHSA-wv4w-6qv2-qqfg:pkg:pypi/social-auth-app-django" + }, + { + "scheme": "semver", + "type": "exact", + "min": null, + "minInclusive": null, + "max": null, + "maxInclusive": null, + "value": "5.4.0", + "notes": "osv:PyPI:GHSA-wv4w-6qv2-qqfg:pkg:pypi/social-auth-app-django" + }, + { + "scheme": "semver", + "type": "exact", + "min": null, + "minInclusive": null, + "max": null, + "maxInclusive": null, + "value": "5.4.1", + "notes": "osv:PyPI:GHSA-wv4w-6qv2-qqfg:pkg:pypi/social-auth-app-django" + }, + { + "scheme": "semver", + "type": "exact", + "min": null, + "minInclusive": null, + "max": null, + "maxInclusive": null, + "value": "5.4.2", + "notes": "osv:PyPI:GHSA-wv4w-6qv2-qqfg:pkg:pypi/social-auth-app-django" + }, + { + "scheme": "semver", + "type": "exact", + "min": null, + "minInclusive": null, + "max": null, + "maxInclusive": null, + "value": "5.4.3", + "notes": "osv:PyPI:GHSA-wv4w-6qv2-qqfg:pkg:pypi/social-auth-app-django" + }, + { + "scheme": "semver", + "type": "exact", + "min": null, + "minInclusive": null, + "max": null, + "maxInclusive": null, + "value": "5.5.0", + "notes": "osv:PyPI:GHSA-wv4w-6qv2-qqfg:pkg:pypi/social-auth-app-django" + }, + { + "scheme": "semver", + "type": "exact", + "min": null, + "minInclusive": null, + "max": null, + "maxInclusive": null, + "value": "5.5.1", + "notes": "osv:PyPI:GHSA-wv4w-6qv2-qqfg:pkg:pypi/social-auth-app-django" + }, + { + "scheme": "semver", + "type": "range", + "min": "0", + "minInclusive": true, + "max": "5.6.0", + "maxInclusive": false, + "value": null, + "notes": "osv:PyPI:GHSA-wv4w-6qv2-qqfg:pkg:pypi/social-auth-app-django" + } + ], + "statuses": [], + "provenance": [ + { + "source": "osv", + "kind": "affected", + "value": "pkg:pypi/social-auth-app-django", + "decisionReason": "source-default", + "recordedAt": "2025-10-09T17:57:29.916841+00:00", + "fieldMask": [] + } + ] + } + ], + "aliases": [ + "CVE-2025-61783", + "GHSA-wv4w-6qv2-qqfg" + ], + "cvssMetrics": [], + "exploitKnown": false, + "language": "en", + "modified": "2025-10-09T17:57:29.916841+00:00", + "provenance": [ + { + "source": "osv", + "kind": "document", + "value": "https://osv.dev/vulnerability/GHSA-wv4w-6qv2-qqfg", + "decisionReason": "source-default", + "recordedAt": "2025-10-09T17:08:05+00:00", + "fieldMask": [] + }, + { + "source": "osv", + "kind": "mapping", + "value": "GHSA-wv4w-6qv2-qqfg", + "decisionReason": "source-default", + "recordedAt": "2025-10-09T17:57:29.916841+00:00", + "fieldMask": [] + } + ], + "published": "2025-10-09T17:08:05+00:00", + "references": [ + { + "kind": null, + "provenance": { + "source": "osv", + "kind": "reference", + "value": "https://github.com/python-social-auth/social-app-django", + "decisionReason": "source-default", + "recordedAt": "2025-10-09T17:57:29.916841+00:00", + "fieldMask": [] + }, + "sourceTag": "PACKAGE", + "summary": null, + "url": "https://github.com/python-social-auth/social-app-django" + }, + { + "kind": null, + "provenance": { + "source": "osv", + "kind": "reference", + "value": "https://github.com/python-social-auth/social-app-django/commit/10c80e2ebabeccd4e9c84ad0e16e1db74148ed4c", + "decisionReason": "source-default", + "recordedAt": "2025-10-09T17:57:29.916841+00:00", + "fieldMask": [] + }, + "sourceTag": "WEB", + "summary": null, + "url": "https://github.com/python-social-auth/social-app-django/commit/10c80e2ebabeccd4e9c84ad0e16e1db74148ed4c" + }, + { + "kind": null, + "provenance": { + "source": "osv", + "kind": "reference", + "value": "https://github.com/python-social-auth/social-app-django/issues/220", + "decisionReason": "source-default", + "recordedAt": "2025-10-09T17:57:29.916841+00:00", + "fieldMask": [] + }, + "sourceTag": "WEB", + "summary": null, + "url": "https://github.com/python-social-auth/social-app-django/issues/220" + }, + { + "kind": null, + "provenance": { + "source": "osv", + "kind": "reference", + "value": "https://github.com/python-social-auth/social-app-django/issues/231", + "decisionReason": "source-default", + "recordedAt": "2025-10-09T17:57:29.916841+00:00", + "fieldMask": [] + }, + "sourceTag": "WEB", + "summary": null, + "url": "https://github.com/python-social-auth/social-app-django/issues/231" + }, + { + "kind": null, + "provenance": { + "source": "osv", + "kind": "reference", + "value": "https://github.com/python-social-auth/social-app-django/issues/634", + "decisionReason": "source-default", + "recordedAt": "2025-10-09T17:57:29.916841+00:00", + "fieldMask": [] + }, + "sourceTag": "WEB", + "summary": null, + "url": "https://github.com/python-social-auth/social-app-django/issues/634" + }, + { + "kind": null, + "provenance": { + "source": "osv", + "kind": "reference", + "value": "https://github.com/python-social-auth/social-app-django/pull/803", + "decisionReason": "source-default", + "recordedAt": "2025-10-09T17:57:29.916841+00:00", + "fieldMask": [] + }, + "sourceTag": "WEB", + "summary": null, + "url": "https://github.com/python-social-auth/social-app-django/pull/803" + }, + { + "kind": null, + "provenance": { + "source": "osv", + "kind": "reference", + "value": "https://github.com/python-social-auth/social-app-django/security/advisories/GHSA-wv4w-6qv2-qqfg", + "decisionReason": "source-default", + "recordedAt": "2025-10-09T17:57:29.916841+00:00", + "fieldMask": [] + }, + "sourceTag": "WEB", + "summary": null, + "url": "https://github.com/python-social-auth/social-app-django/security/advisories/GHSA-wv4w-6qv2-qqfg" + } + ], + "severity": "medium", + "summary": "### Impact Upon authentication, the user could be associated by e-mail even if the `associate_by_email` pipeline was not included. This could lead to account compromise when a third-party authentication service does not validate provided e-mail addresses or doesn't require unique e-mail addresses. ### Patches * https://github.com/python-social-auth/social-app-django/pull/803 ### Workarounds Review the authentication service policy on e-mail addresses; many will not allow exploiting this vulnerability.", + "title": "Python Social Auth - Django has unsafe account association" + } +] \ No newline at end of file diff --git a/src/StellaOps.Feedser.Source.Osv.Tests/Fixtures/osv-ghsa.raw-ghsa.json b/src/StellaOps.Feedser.Source.Osv.Tests/Fixtures/osv-ghsa.raw-ghsa.json new file mode 100644 index 00000000..4d2d84b1 --- /dev/null +++ b/src/StellaOps.Feedser.Source.Osv.Tests/Fixtures/osv-ghsa.raw-ghsa.json @@ -0,0 +1,519 @@ +[ + { + "ghsa_id": "GHSA-wv4w-6qv2-qqfg", + "cve_id": "CVE-2025-61783", + "url": "https://api.github.com/advisories/GHSA-wv4w-6qv2-qqfg", + "html_url": "https://github.com/advisories/GHSA-wv4w-6qv2-qqfg", + "summary": "Python Social Auth - Django has unsafe account association ", + "description": "### Impact\n\nUpon authentication, the user could be associated by e-mail even if the \u0060associate_by_email\u0060 pipeline was not included. This could lead to account compromise when a third-party authentication service does not validate provided e-mail addresses or doesn\u0027t require unique e-mail addresses.\n\n### Patches\n\n* https://github.com/python-social-auth/social-app-django/pull/803\n\n### Workarounds\n\nReview the authentication service policy on e-mail addresses; many will not allow exploiting this vulnerability.", + "type": "reviewed", + "severity": "medium", + "repository_advisory_url": "https://api.github.com/repos/python-social-auth/social-app-django/security-advisories/GHSA-wv4w-6qv2-qqfg", + "source_code_location": "https://github.com/python-social-auth/social-app-django", + "identifiers": [ + { + "value": "GHSA-wv4w-6qv2-qqfg", + "type": "GHSA" + }, + { + "value": "CVE-2025-61783", + "type": "CVE" + } + ], + "references": [ + "https://github.com/python-social-auth/social-app-django/security/advisories/GHSA-wv4w-6qv2-qqfg", + "https://github.com/python-social-auth/social-app-django/issues/220", + "https://github.com/python-social-auth/social-app-django/issues/231", + "https://github.com/python-social-auth/social-app-django/issues/634", + "https://github.com/python-social-auth/social-app-django/pull/803", + "https://github.com/python-social-auth/social-app-django/commit/10c80e2ebabeccd4e9c84ad0e16e1db74148ed4c", + "https://github.com/advisories/GHSA-wv4w-6qv2-qqfg" + ], + "published_at": "2025-10-09T17:08:05Z", + "updated_at": "2025-10-09T17:08:06Z", + "github_reviewed_at": "2025-10-09T17:08:05Z", + "nvd_published_at": null, + "withdrawn_at": null, + "vulnerabilities": [ + { + "package": { + "ecosystem": "pip", + "name": "social-auth-app-django" + }, + "vulnerable_version_range": "\u003C 5.6.0", + "first_patched_version": "5.6.0", + "vulnerable_functions": [] + } + ], + "cvss_severities": { + "cvss_v3": { + "vector_string": null, + "score": 0.0 + }, + "cvss_v4": { + "vector_string": "CVSS:4.0/AV:N/AC:H/AT:N/PR:N/UI:N/VC:L/VI:L/VA:N/SC:N/SI:N/SA:N", + "score": 6.3 + } + }, + "cwes": [ + { + "cwe_id": "CWE-290", + "name": "Authentication Bypass by Spoofing" + } + ], + "credits": [ + { + "user": { + "login": "mel-mason", + "id": 19391457, + "node_id": "MDQ6VXNlcjE5MzkxNDU3", + "avatar_url": "https://avatars.githubusercontent.com/u/19391457?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/mel-mason", + "html_url": "https://github.com/mel-mason", + "followers_url": "https://api.github.com/users/mel-mason/followers", + "following_url": "https://api.github.com/users/mel-mason/following{/other_user}", + "gists_url": "https://api.github.com/users/mel-mason/gists{/gist_id}", + "starred_url": "https://api.github.com/users/mel-mason/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/mel-mason/subscriptions", + "organizations_url": "https://api.github.com/users/mel-mason/orgs", + "repos_url": "https://api.github.com/users/mel-mason/repos", + "events_url": "https://api.github.com/users/mel-mason/events{/privacy}", + "received_events_url": "https://api.github.com/users/mel-mason/received_events", + "type": "User", + "user_view_type": "public", + "site_admin": false + }, + "type": "reporter" + }, + { + "user": { + "login": "vanya909", + "id": 53380238, + "node_id": "MDQ6VXNlcjUzMzgwMjM4", + "avatar_url": "https://avatars.githubusercontent.com/u/53380238?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/vanya909", + "html_url": "https://github.com/vanya909", + "followers_url": "https://api.github.com/users/vanya909/followers", + "following_url": "https://api.github.com/users/vanya909/following{/other_user}", + "gists_url": "https://api.github.com/users/vanya909/gists{/gist_id}", + "starred_url": "https://api.github.com/users/vanya909/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/vanya909/subscriptions", + "organizations_url": "https://api.github.com/users/vanya909/orgs", + "repos_url": "https://api.github.com/users/vanya909/repos", + "events_url": "https://api.github.com/users/vanya909/events{/privacy}", + "received_events_url": "https://api.github.com/users/vanya909/received_events", + "type": "User", + "user_view_type": "public", + "site_admin": false + }, + "type": "reporter" + }, + { + "user": { + "login": "nijel", + "id": 212189, + "node_id": "MDQ6VXNlcjIxMjE4OQ==", + "avatar_url": "https://avatars.githubusercontent.com/u/212189?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/nijel", + "html_url": "https://github.com/nijel", + "followers_url": "https://api.github.com/users/nijel/followers", + "following_url": "https://api.github.com/users/nijel/following{/other_user}", + "gists_url": "https://api.github.com/users/nijel/gists{/gist_id}", + "starred_url": "https://api.github.com/users/nijel/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/nijel/subscriptions", + "organizations_url": "https://api.github.com/users/nijel/orgs", + "repos_url": "https://api.github.com/users/nijel/repos", + "events_url": "https://api.github.com/users/nijel/events{/privacy}", + "received_events_url": "https://api.github.com/users/nijel/received_events", + "type": "User", + "user_view_type": "public", + "site_admin": false + }, + "type": "remediation_developer" + } + ], + "cvss": { + "vector_string": null, + "score": null + } + }, + { + "ghsa_id": "GHSA-cjjf-27cc-pvmv", + "cve_id": "CVE-2025-61773", + "url": "https://api.github.com/advisories/GHSA-cjjf-27cc-pvmv", + "html_url": "https://github.com/advisories/GHSA-cjjf-27cc-pvmv", + "summary": "pyLoad CNL and captcha handlers allow Code Injection via unsanitized parameters", + "description": "### Summary\npyLoad web interface contained insufficient input validation in both the Captcha script endpoint and the Click\u0027N\u0027Load (CNL) Blueprint. This flaw allowed untrusted user input to be processed unsafely, which could be exploited by an attacker to inject arbitrary content into the web UI or manipulate request handling. The vulnerability could lead to client-side code execution (XSS) or other unintended behaviors when a malicious payload is submitted.\n\nuser-supplied parameters from HTTP requests were not adequately validated or sanitized before being passed into the application logic and response generation. This allowed crafted input to alter the expected execution flow.\n CNL (Click\u0027N\u0027Load) blueprint exposed unsafe handling of untrusted parameters in HTTP requests. The application did not consistently enforce input validation or encoding, making it possible for an attacker to craft malicious requests.\n\n### PoC\n\n1. Run a vulnerable version of pyLoad prior to commit [\u0060f9d27f2\u0060](https://github.com/pyload/pyload/pull/4624).\n2. Start the web UI and access the Captcha or CNL endpoints.\n3. Submit a crafted request containing malicious JavaScript payloads in unvalidated parameters (\u0060/flash/addcrypted2?jk=function(){alert(1)}\u0026crypted=12345\u0060).\n4. Observe that the payload is reflected and executed in the client\u2019s browser, demonstrating cross-site scripting (XSS).\n\nExample request:\n\n\u0060\u0060\u0060http\nGET /flash/addcrypted2?jk=function(){alert(1)}\u0026crypted=12345 HTTP/1.1\nHost: 127.0.0.1:8000\nContent-Type: application/x-www-form-urlencoded\nContent-Length: 107\n\u0060\u0060\u0060\n\n### Impact\n\nExploiting this vulnerability allows an attacker to inject and execute arbitrary JavaScript within the browser session of a user accessing the pyLoad Web UI. In practice, this means an attacker could impersonate an administrator, steal authentication cookies or tokens, and perform unauthorized actions on behalf of the victim. Because the affected endpoints are part of the core interface, a successful attack undermines the trust and security of the entire application, potentially leading to a full compromise of the management interface and the data it controls. The impact is particularly severe in cases where the Web UI is exposed over a network without additional access restrictions, as it enables remote attackers to directly target users with crafted links or requests that trigger the vulnerability.", + "type": "reviewed", + "severity": "high", + "repository_advisory_url": "https://api.github.com/repos/pyload/pyload/security-advisories/GHSA-cjjf-27cc-pvmv", + "source_code_location": "https://github.com/pyload/pyload", + "identifiers": [ + { + "value": "GHSA-cjjf-27cc-pvmv", + "type": "GHSA" + }, + { + "value": "CVE-2025-61773", + "type": "CVE" + } + ], + "references": [ + "https://github.com/pyload/pyload/security/advisories/GHSA-cjjf-27cc-pvmv", + "https://github.com/pyload/pyload/pull/4624", + "https://github.com/pyload/pyload/commit/5823327d0b797161c7195a1f660266d30a69f0ca", + "https://github.com/advisories/GHSA-cjjf-27cc-pvmv" + ], + "published_at": "2025-10-09T15:19:48Z", + "updated_at": "2025-10-09T15:19:48Z", + "github_reviewed_at": "2025-10-09T15:19:48Z", + "nvd_published_at": null, + "withdrawn_at": null, + "vulnerabilities": [ + { + "package": { + "ecosystem": "pip", + "name": "pyload-ng" + }, + "vulnerable_version_range": "\u003C 0.5.0b3.dev91", + "first_patched_version": "0.5.0b3.dev91", + "vulnerable_functions": [] + } + ], + "cvss_severities": { + "cvss_v3": { + "vector_string": "CVSS:3.1/AV:N/AC:L/PR:N/UI:R/S:U/C:H/I:H/A:N", + "score": 8.1 + }, + "cvss_v4": { + "vector_string": null, + "score": 0.0 + } + }, + "cwes": [ + { + "cwe_id": "CWE-74", + "name": "Improper Neutralization of Special Elements in Output Used by a Downstream Component (\u0027Injection\u0027)" + }, + { + "cwe_id": "CWE-79", + "name": "Improper Neutralization of Input During Web Page Generation (\u0027Cross-site Scripting\u0027)" + }, + { + "cwe_id": "CWE-94", + "name": "Improper Control of Generation of Code (\u0027Code Injection\u0027)" + }, + { + "cwe_id": "CWE-116", + "name": "Improper Encoding or Escaping of Output" + } + ], + "credits": [ + { + "user": { + "login": "odaysec", + "id": 47859767, + "node_id": "MDQ6VXNlcjQ3ODU5NzY3", + "avatar_url": "https://avatars.githubusercontent.com/u/47859767?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/odaysec", + "html_url": "https://github.com/odaysec", + "followers_url": "https://api.github.com/users/odaysec/followers", + "following_url": "https://api.github.com/users/odaysec/following{/other_user}", + "gists_url": "https://api.github.com/users/odaysec/gists{/gist_id}", + "starred_url": "https://api.github.com/users/odaysec/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/odaysec/subscriptions", + "organizations_url": "https://api.github.com/users/odaysec/orgs", + "repos_url": "https://api.github.com/users/odaysec/repos", + "events_url": "https://api.github.com/users/odaysec/events{/privacy}", + "received_events_url": "https://api.github.com/users/odaysec/received_events", + "type": "User", + "user_view_type": "public", + "site_admin": false + }, + "type": "reporter" + } + ], + "cvss": { + "vector_string": "CVSS:3.1/AV:N/AC:L/PR:N/UI:R/S:U/C:H/I:H/A:N", + "score": 8.1 + } + }, + { + "ghsa_id": "GHSA-77vh-xpmg-72qh", + "cve_id": null, + "url": "https://api.github.com/advisories/GHSA-77vh-xpmg-72qh", + "html_url": "https://github.com/advisories/GHSA-77vh-xpmg-72qh", + "summary": "Clarify \u0060mediaType\u0060 handling", + "description": "### Impact\nIn the OCI Image Specification version 1.0.1 and prior, manifest and index documents are not self-describing and documents with a single digest could be interpreted as either a manifest or an index.\n\n### Patches\nThe Image Specification will be updated to recommend that both manifest and index documents contain a \u0060mediaType\u0060 field to identify the type of document.\nRelease [v1.0.2](https://github.com/opencontainers/image-spec/releases/tag/v1.0.2) includes these updates.\n\n### Workarounds\nSoftware attempting to deserialize an ambiguous document may reject the document if it contains both \u201Cmanifests\u201D and \u201Clayers\u201D fields or \u201Cmanifests\u201D and \u201Cconfig\u201D fields.\n\n### References\nhttps://github.com/opencontainers/distribution-spec/security/advisories/GHSA-mc8v-mgrf-8f4m\n\n### For more information\nIf you have any questions or comments about this advisory:\n* Open an issue in https://github.com/opencontainers/image-spec\n* Email us at [security@opencontainers.org](mailto:security@opencontainers.org)\n* https://github.com/opencontainers/image-spec/commits/v1.0.2\n", + "type": "reviewed", + "severity": "low", + "repository_advisory_url": "https://api.github.com/repos/opencontainers/image-spec/security-advisories/GHSA-77vh-xpmg-72qh", + "source_code_location": "https://github.com/opencontainers/image-spec", + "identifiers": [ + { + "value": "GHSA-77vh-xpmg-72qh", + "type": "GHSA" + } + ], + "references": [ + "https://github.com/opencontainers/distribution-spec/security/advisories/GHSA-mc8v-mgrf-8f4m", + "https://github.com/opencontainers/image-spec/security/advisories/GHSA-77vh-xpmg-72qh", + "https://github.com/opencontainers/image-spec/commit/693428a734f5bab1a84bd2f990d92ef1111cd60c", + "https://github.com/opencontainers/image-spec/releases/tag/v1.0.2", + "https://github.com/advisories/GHSA-77vh-xpmg-72qh" + ], + "published_at": "2021-11-18T16:02:41Z", + "updated_at": "2023-01-09T05:05:32Z", + "github_reviewed_at": "2021-11-17T23:13:41Z", + "nvd_published_at": null, + "withdrawn_at": null, + "vulnerabilities": [ + { + "package": { + "ecosystem": "go", + "name": "github.com/opencontainers/image-spec" + }, + "vulnerable_version_range": "\u003C 1.0.2", + "first_patched_version": "1.0.2", + "vulnerable_functions": [] + } + ], + "cvss_severities": { + "cvss_v3": { + "vector_string": "CVSS:3.1/AV:N/AC:H/PR:L/UI:R/S:C/C:N/I:L/A:N", + "score": 3.0 + }, + "cvss_v4": { + "vector_string": null, + "score": 0.0 + } + }, + "cwes": [ + { + "cwe_id": "CWE-843", + "name": "Access of Resource Using Incompatible Type (\u0027Type Confusion\u0027)" + } + ], + "credits": [], + "cvss": { + "vector_string": "CVSS:3.1/AV:N/AC:H/PR:L/UI:R/S:C/C:N/I:L/A:N", + "score": 3.0 + } + }, + { + "ghsa_id": "GHSA-7rjr-3q55-vv33", + "cve_id": "CVE-2021-45046", + "url": "https://api.github.com/advisories/GHSA-7rjr-3q55-vv33", + "html_url": "https://github.com/advisories/GHSA-7rjr-3q55-vv33", + "summary": "Incomplete fix for Apache Log4j vulnerability", + "description": "# Impact\n\nThe fix to address [CVE-2021-44228](https://nvd.nist.gov/vuln/detail/CVE-2021-44228) in Apache Log4j 2.15.0 was incomplete in certain non-default configurations. This could allow attackers with control over Thread Context Map (MDC) input data when the logging configuration uses a non-default Pattern Layout with either a Context Lookup (for example, $${ctx:loginId}) or a Thread Context Map pattern (%X, %mdc, or %MDC) to craft malicious input data using a JNDI Lookup pattern resulting in a remote code execution (RCE) attack. \n\n## Affected packages\nOnly the \u0060org.apache.logging.log4j:log4j-core\u0060 package is directly affected by this vulnerability. The \u0060org.apache.logging.log4j:log4j-api\u0060 should be kept at the same version as the \u0060org.apache.logging.log4j:log4j-core\u0060 package to ensure compatability if in use.\n\n# Mitigation\n\nLog4j 2.16.0 fixes this issue by removing support for message lookup patterns and disabling JNDI functionality by default. This issue can be mitigated in prior releases (\u003C 2.16.0) by removing the JndiLookup class from the classpath (example: zip -q -d log4j-core-*.jar org/apache/logging/log4j/core/lookup/JndiLookup.class).\n\nLog4j 2.15.0 restricts JNDI LDAP lookups to localhost by default. Note that previous mitigations involving configuration such as to set the system property \u0060log4j2.formatMsgNoLookups\u0060 to \u0060true\u0060 do NOT mitigate this specific vulnerability.", + "type": "reviewed", + "severity": "critical", + "repository_advisory_url": null, + "source_code_location": "", + "identifiers": [ + { + "value": "GHSA-7rjr-3q55-vv33", + "type": "GHSA" + }, + { + "value": "CVE-2021-45046", + "type": "CVE" + } + ], + "references": [ + "https://nvd.nist.gov/vuln/detail/CVE-2021-45046", + "https://github.com/advisories/GHSA-jfh8-c2jp-5v3q", + "https://logging.apache.org/log4j/2.x/security.html", + "https://www.openwall.com/lists/oss-security/2021/12/14/4", + "https://www.cve.org/CVERecord?id=CVE-2021-44228", + "http://www.openwall.com/lists/oss-security/2021/12/14/4", + "https://www.intel.com/content/www/us/en/security-center/advisory/intel-sa-00646.html", + "http://www.openwall.com/lists/oss-security/2021/12/15/3", + "https://cert-portal.siemens.com/productcert/pdf/ssa-661247.pdf", + "https://cert-portal.siemens.com/productcert/pdf/ssa-714170.pdf", + "https://www.kb.cert.org/vuls/id/930724", + "https://www.debian.org/security/2021/dsa-5022", + "https://psirt.global.sonicwall.com/vuln-detail/SNWLID-2021-0032", + "https://www.oracle.com/security-alerts/alert-cve-2021-44228.html", + "http://www.openwall.com/lists/oss-security/2021/12/18/1", + "https://cert-portal.siemens.com/productcert/pdf/ssa-397453.pdf", + "https://cert-portal.siemens.com/productcert/pdf/ssa-479842.pdf", + "https://www.oracle.com/security-alerts/cpujan2022.html", + "https://www.oracle.com/security-alerts/cpuapr2022.html", + "https://www.oracle.com/security-alerts/cpujul2022.html", + "https://security.gentoo.org/glsa/202310-16", + "https://lists.fedoraproject.org/archives/list/package-announce@lists.fedoraproject.org/message/SIG7FZULMNK2XF6FZRU4VWYDQXNMUGAJ", + "https://lists.fedoraproject.org/archives/list/package-announce@lists.fedoraproject.org/message/EOKPQGV24RRBBI4TBZUDQMM4MEH7MXCY", + "https://sec.cloudapps.cisco.com/security/center/content/CiscoSecurityAdvisory/cisco-sa-apache-log4j-qRuKNEbd", + "https://github.com/advisories/GHSA-7rjr-3q55-vv33" + ], + "published_at": "2021-12-14T18:01:28Z", + "updated_at": "2025-05-09T12:28:41Z", + "github_reviewed_at": "2021-12-14T17:55:00Z", + "nvd_published_at": "2021-12-14T19:15:00Z", + "withdrawn_at": null, + "vulnerabilities": [ + { + "package": { + "ecosystem": "maven", + "name": "org.apache.logging.log4j:log4j-core" + }, + "vulnerable_version_range": "\u003E= 2.13.0, \u003C 2.16.0", + "first_patched_version": "2.16.0", + "vulnerable_functions": [] + }, + { + "package": { + "ecosystem": "maven", + "name": "org.apache.logging.log4j:log4j-core" + }, + "vulnerable_version_range": "\u003C 2.12.2", + "first_patched_version": "2.12.2", + "vulnerable_functions": [] + }, + { + "package": { + "ecosystem": "maven", + "name": "org.ops4j.pax.logging:pax-logging-log4j2" + }, + "vulnerable_version_range": "\u003E= 1.8.0, \u003C 1.9.2", + "first_patched_version": "1.9.2", + "vulnerable_functions": [] + }, + { + "package": { + "ecosystem": "maven", + "name": "org.ops4j.pax.logging:pax-logging-log4j2" + }, + "vulnerable_version_range": "\u003E= 1.10.0, \u003C 1.10.8", + "first_patched_version": "1.10.8", + "vulnerable_functions": [] + }, + { + "package": { + "ecosystem": "maven", + "name": "org.ops4j.pax.logging:pax-logging-log4j2" + }, + "vulnerable_version_range": "\u003E= 1.11.0, \u003C 1.11.11", + "first_patched_version": "1.11.11", + "vulnerable_functions": [] + }, + { + "package": { + "ecosystem": "maven", + "name": "org.ops4j.pax.logging:pax-logging-log4j2" + }, + "vulnerable_version_range": "\u003E= 2.0.0, \u003C 2.0.12", + "first_patched_version": "2.0.12", + "vulnerable_functions": [] + } + ], + "cvss_severities": { + "cvss_v3": { + "vector_string": "CVSS:3.1/AV:N/AC:H/PR:N/UI:N/S:C/C:H/I:H/A:H", + "score": 9.1 + }, + "cvss_v4": { + "vector_string": null, + "score": 0.0 + } + }, + "cwes": [ + { + "cwe_id": "CWE-502", + "name": "Deserialization of Untrusted Data" + }, + { + "cwe_id": "CWE-917", + "name": "Improper Neutralization of Special Elements used in an Expression Language Statement (\u0027Expression Language Injection\u0027)" + } + ], + "credits": [ + { + "user": { + "login": "mrjonstrong", + "id": 42520909, + "node_id": "MDQ6VXNlcjQyNTIwOTA5", + "avatar_url": "https://avatars.githubusercontent.com/u/42520909?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/mrjonstrong", + "html_url": "https://github.com/mrjonstrong", + "followers_url": "https://api.github.com/users/mrjonstrong/followers", + "following_url": "https://api.github.com/users/mrjonstrong/following{/other_user}", + "gists_url": "https://api.github.com/users/mrjonstrong/gists{/gist_id}", + "starred_url": "https://api.github.com/users/mrjonstrong/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/mrjonstrong/subscriptions", + "organizations_url": "https://api.github.com/users/mrjonstrong/orgs", + "repos_url": "https://api.github.com/users/mrjonstrong/repos", + "events_url": "https://api.github.com/users/mrjonstrong/events{/privacy}", + "received_events_url": "https://api.github.com/users/mrjonstrong/received_events", + "type": "User", + "user_view_type": "public", + "site_admin": false + }, + "type": "analyst" + }, + { + "user": { + "login": "afdesk", + "id": 19297627, + "node_id": "MDQ6VXNlcjE5Mjk3NjI3", + "avatar_url": "https://avatars.githubusercontent.com/u/19297627?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/afdesk", + "html_url": "https://github.com/afdesk", + "followers_url": "https://api.github.com/users/afdesk/followers", + "following_url": "https://api.github.com/users/afdesk/following{/other_user}", + "gists_url": "https://api.github.com/users/afdesk/gists{/gist_id}", + "starred_url": "https://api.github.com/users/afdesk/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/afdesk/subscriptions", + "organizations_url": "https://api.github.com/users/afdesk/orgs", + "repos_url": "https://api.github.com/users/afdesk/repos", + "events_url": "https://api.github.com/users/afdesk/events{/privacy}", + "received_events_url": "https://api.github.com/users/afdesk/received_events", + "type": "User", + "user_view_type": "public", + "site_admin": false + }, + "type": "analyst" + }, + { + "user": { + "login": "ppkarwasz", + "id": 12533274, + "node_id": "MDQ6VXNlcjEyNTMzMjc0", + "avatar_url": "https://avatars.githubusercontent.com/u/12533274?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/ppkarwasz", + "html_url": "https://github.com/ppkarwasz", + "followers_url": "https://api.github.com/users/ppkarwasz/followers", + "following_url": "https://api.github.com/users/ppkarwasz/following{/other_user}", + "gists_url": "https://api.github.com/users/ppkarwasz/gists{/gist_id}", + "starred_url": "https://api.github.com/users/ppkarwasz/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/ppkarwasz/subscriptions", + "organizations_url": "https://api.github.com/users/ppkarwasz/orgs", + "repos_url": "https://api.github.com/users/ppkarwasz/repos", + "events_url": "https://api.github.com/users/ppkarwasz/events{/privacy}", + "received_events_url": "https://api.github.com/users/ppkarwasz/received_events", + "type": "User", + "user_view_type": "public", + "site_admin": false + }, + "type": "analyst" + } + ], + "cvss": { + "vector_string": "CVSS:3.1/AV:N/AC:H/PR:N/UI:N/S:C/C:H/I:H/A:H", + "score": 9.1 + }, + "epss": { + "percentage": 0.9434, + "percentile": 0.9995 + } + } +] \ No newline at end of file diff --git a/src/StellaOps.Feedser.Source.Osv.Tests/Fixtures/osv-ghsa.raw-osv.json b/src/StellaOps.Feedser.Source.Osv.Tests/Fixtures/osv-ghsa.raw-osv.json new file mode 100644 index 00000000..b48295ab --- /dev/null +++ b/src/StellaOps.Feedser.Source.Osv.Tests/Fixtures/osv-ghsa.raw-osv.json @@ -0,0 +1,714 @@ +[ + { + "id": "GHSA-wv4w-6qv2-qqfg", + "summary": "Python Social Auth - Django has unsafe account association ", + "details": "### Impact\n\nUpon authentication, the user could be associated by e-mail even if the \u0060associate_by_email\u0060 pipeline was not included. This could lead to account compromise when a third-party authentication service does not validate provided e-mail addresses or doesn\u0027t require unique e-mail addresses.\n\n### Patches\n\n* https://github.com/python-social-auth/social-app-django/pull/803\n\n### Workarounds\n\nReview the authentication service policy on e-mail addresses; many will not allow exploiting this vulnerability.", + "aliases": [ + "CVE-2025-61783" + ], + "modified": "2025-10-09T17:57:29.916841Z", + "published": "2025-10-09T17:08:05Z", + "database_specific": { + "github_reviewed_at": "2025-10-09T17:08:05Z", + "severity": "MODERATE", + "cwe_ids": [ + "CWE-290" + ], + "github_reviewed": true, + "nvd_published_at": null + }, + "references": [ + { + "type": "WEB", + "url": "https://github.com/python-social-auth/social-app-django/security/advisories/GHSA-wv4w-6qv2-qqfg" + }, + { + "type": "WEB", + "url": "https://github.com/python-social-auth/social-app-django/issues/220" + }, + { + "type": "WEB", + "url": "https://github.com/python-social-auth/social-app-django/issues/231" + }, + { + "type": "WEB", + "url": "https://github.com/python-social-auth/social-app-django/issues/634" + }, + { + "type": "WEB", + "url": "https://github.com/python-social-auth/social-app-django/pull/803" + }, + { + "type": "WEB", + "url": "https://github.com/python-social-auth/social-app-django/commit/10c80e2ebabeccd4e9c84ad0e16e1db74148ed4c" + }, + { + "type": "PACKAGE", + "url": "https://github.com/python-social-auth/social-app-django" + } + ], + "affected": [ + { + "package": { + "name": "social-auth-app-django", + "ecosystem": "PyPI", + "purl": "pkg:pypi/social-auth-app-django" + }, + "ranges": [ + { + "type": "ECOSYSTEM", + "events": [ + { + "introduced": "0" + }, + { + "fixed": "5.6.0" + } + ] + } + ], + "versions": [ + "0.0.1", + "0.1.0", + "1.0.0", + "1.0.1", + "1.1.0", + "1.2.0", + "2.0.0", + "2.1.0", + "3.0.0", + "3.1.0", + "3.3.0", + "3.4.0", + "4.0.0", + "5.0.0", + "5.1.0", + "5.2.0", + "5.3.0", + "5.4.0", + "5.4.1", + "5.4.2", + "5.4.3", + "5.5.0", + "5.5.1" + ], + "database_specific": { + "source": "https://github.com/github/advisory-database/blob/main/advisories/github-reviewed/2025/10/GHSA-wv4w-6qv2-qqfg/GHSA-wv4w-6qv2-qqfg.json" + } + } + ], + "schema_version": "1.7.3", + "severity": [ + { + "type": "CVSS_V4", + "score": "CVSS:4.0/AV:N/AC:H/AT:N/PR:N/UI:N/VC:L/VI:L/VA:N/SC:N/SI:N/SA:N" + } + ] + }, + { + "id": "GHSA-cjjf-27cc-pvmv", + "summary": "pyLoad CNL and captcha handlers allow Code Injection via unsanitized parameters", + "details": "### Summary\npyLoad web interface contained insufficient input validation in both the Captcha script endpoint and the Click\u0027N\u0027Load (CNL) Blueprint. This flaw allowed untrusted user input to be processed unsafely, which could be exploited by an attacker to inject arbitrary content into the web UI or manipulate request handling. The vulnerability could lead to client-side code execution (XSS) or other unintended behaviors when a malicious payload is submitted.\n\nuser-supplied parameters from HTTP requests were not adequately validated or sanitized before being passed into the application logic and response generation. This allowed crafted input to alter the expected execution flow.\n CNL (Click\u0027N\u0027Load) blueprint exposed unsafe handling of untrusted parameters in HTTP requests. The application did not consistently enforce input validation or encoding, making it possible for an attacker to craft malicious requests.\n\n### PoC\n\n1. Run a vulnerable version of pyLoad prior to commit [\u0060f9d27f2\u0060](https://github.com/pyload/pyload/pull/4624).\n2. Start the web UI and access the Captcha or CNL endpoints.\n3. Submit a crafted request containing malicious JavaScript payloads in unvalidated parameters (\u0060/flash/addcrypted2?jk=function(){alert(1)}\u0026crypted=12345\u0060).\n4. Observe that the payload is reflected and executed in the client\u2019s browser, demonstrating cross-site scripting (XSS).\n\nExample request:\n\n\u0060\u0060\u0060http\nGET /flash/addcrypted2?jk=function(){alert(1)}\u0026crypted=12345 HTTP/1.1\nHost: 127.0.0.1:8000\nContent-Type: application/x-www-form-urlencoded\nContent-Length: 107\n\u0060\u0060\u0060\n\n### Impact\n\nExploiting this vulnerability allows an attacker to inject and execute arbitrary JavaScript within the browser session of a user accessing the pyLoad Web UI. In practice, this means an attacker could impersonate an administrator, steal authentication cookies or tokens, and perform unauthorized actions on behalf of the victim. Because the affected endpoints are part of the core interface, a successful attack undermines the trust and security of the entire application, potentially leading to a full compromise of the management interface and the data it controls. The impact is particularly severe in cases where the Web UI is exposed over a network without additional access restrictions, as it enables remote attackers to directly target users with crafted links or requests that trigger the vulnerability.", + "aliases": [ + "CVE-2025-61773" + ], + "modified": "2025-10-09T15:59:13.250015Z", + "published": "2025-10-09T15:19:48Z", + "database_specific": { + "github_reviewed_at": "2025-10-09T15:19:48Z", + "github_reviewed": true, + "cwe_ids": [ + "CWE-116", + "CWE-74", + "CWE-79", + "CWE-94" + ], + "severity": "HIGH", + "nvd_published_at": null + }, + "references": [ + { + "type": "WEB", + "url": "https://github.com/pyload/pyload/security/advisories/GHSA-cjjf-27cc-pvmv" + }, + { + "type": "WEB", + "url": "https://github.com/pyload/pyload/pull/4624" + }, + { + "type": "WEB", + "url": "https://github.com/pyload/pyload/commit/5823327d0b797161c7195a1f660266d30a69f0ca" + }, + { + "type": "PACKAGE", + "url": "https://github.com/pyload/pyload" + } + ], + "affected": [ + { + "package": { + "name": "pyload-ng", + "ecosystem": "PyPI", + "purl": "pkg:pypi/pyload-ng" + }, + "ranges": [ + { + "type": "ECOSYSTEM", + "events": [ + { + "introduced": "0" + }, + { + "fixed": "0.5.0b3.dev91" + } + ] + } + ], + "versions": [ + "0.5.0a5.dev528", + "0.5.0a5.dev532", + "0.5.0a5.dev535", + "0.5.0a5.dev536", + "0.5.0a5.dev537", + "0.5.0a5.dev539", + "0.5.0a5.dev540", + "0.5.0a5.dev545", + "0.5.0a5.dev562", + "0.5.0a5.dev564", + "0.5.0a5.dev565", + "0.5.0a6.dev570", + "0.5.0a6.dev578", + "0.5.0a6.dev587", + "0.5.0a7.dev596", + "0.5.0a8.dev602", + "0.5.0a9.dev615", + "0.5.0a9.dev629", + "0.5.0a9.dev632", + "0.5.0a9.dev641", + "0.5.0a9.dev643", + "0.5.0a9.dev655", + "0.5.0a9.dev806", + "0.5.0b1.dev1", + "0.5.0b1.dev2", + "0.5.0b1.dev3", + "0.5.0b1.dev4", + "0.5.0b1.dev5", + "0.5.0b2.dev10", + "0.5.0b2.dev11", + "0.5.0b2.dev12", + "0.5.0b2.dev9", + "0.5.0b3.dev13", + "0.5.0b3.dev14", + "0.5.0b3.dev17", + "0.5.0b3.dev18", + "0.5.0b3.dev19", + "0.5.0b3.dev20", + "0.5.0b3.dev21", + "0.5.0b3.dev22", + "0.5.0b3.dev24", + "0.5.0b3.dev26", + "0.5.0b3.dev27", + "0.5.0b3.dev28", + "0.5.0b3.dev29", + "0.5.0b3.dev30", + "0.5.0b3.dev31", + "0.5.0b3.dev32", + "0.5.0b3.dev33", + "0.5.0b3.dev34", + "0.5.0b3.dev35", + "0.5.0b3.dev38", + "0.5.0b3.dev39", + "0.5.0b3.dev40", + "0.5.0b3.dev41", + "0.5.0b3.dev42", + "0.5.0b3.dev43", + "0.5.0b3.dev44", + "0.5.0b3.dev45", + "0.5.0b3.dev46", + "0.5.0b3.dev47", + "0.5.0b3.dev48", + "0.5.0b3.dev49", + "0.5.0b3.dev50", + "0.5.0b3.dev51", + "0.5.0b3.dev52", + "0.5.0b3.dev53", + "0.5.0b3.dev54", + "0.5.0b3.dev57", + "0.5.0b3.dev60", + "0.5.0b3.dev62", + "0.5.0b3.dev64", + "0.5.0b3.dev65", + "0.5.0b3.dev66", + "0.5.0b3.dev67", + "0.5.0b3.dev68", + "0.5.0b3.dev69", + "0.5.0b3.dev70", + "0.5.0b3.dev71", + "0.5.0b3.dev72", + "0.5.0b3.dev73", + "0.5.0b3.dev74", + "0.5.0b3.dev75", + "0.5.0b3.dev76", + "0.5.0b3.dev77", + "0.5.0b3.dev78", + "0.5.0b3.dev79", + "0.5.0b3.dev80", + "0.5.0b3.dev81", + "0.5.0b3.dev82", + "0.5.0b3.dev85", + "0.5.0b3.dev87", + "0.5.0b3.dev88", + "0.5.0b3.dev89", + "0.5.0b3.dev90" + ], + "database_specific": { + "source": "https://github.com/github/advisory-database/blob/main/advisories/github-reviewed/2025/10/GHSA-cjjf-27cc-pvmv/GHSA-cjjf-27cc-pvmv.json" + } + } + ], + "schema_version": "1.7.3", + "severity": [ + { + "type": "CVSS_V3", + "score": "CVSS:3.1/AV:N/AC:L/PR:N/UI:R/S:U/C:H/I:H/A:N" + } + ] + }, + { + "id": "GHSA-77vh-xpmg-72qh", + "summary": "Clarify \u0060mediaType\u0060 handling", + "details": "### Impact\nIn the OCI Image Specification version 1.0.1 and prior, manifest and index documents are not self-describing and documents with a single digest could be interpreted as either a manifest or an index.\n\n### Patches\nThe Image Specification will be updated to recommend that both manifest and index documents contain a \u0060mediaType\u0060 field to identify the type of document.\nRelease [v1.0.2](https://github.com/opencontainers/image-spec/releases/tag/v1.0.2) includes these updates.\n\n### Workarounds\nSoftware attempting to deserialize an ambiguous document may reject the document if it contains both \u201Cmanifests\u201D and \u201Clayers\u201D fields or \u201Cmanifests\u201D and \u201Cconfig\u201D fields.\n\n### References\nhttps://github.com/opencontainers/distribution-spec/security/advisories/GHSA-mc8v-mgrf-8f4m\n\n### For more information\nIf you have any questions or comments about this advisory:\n* Open an issue in https://github.com/opencontainers/image-spec\n* Email us at [security@opencontainers.org](mailto:security@opencontainers.org)\n* https://github.com/opencontainers/image-spec/commits/v1.0.2\n", + "modified": "2021-11-24T19:43:35Z", + "published": "2021-11-18T16:02:41Z", + "related": [ + "CGA-j36r-723f-8c29" + ], + "database_specific": { + "github_reviewed": true, + "nvd_published_at": null, + "github_reviewed_at": "2021-11-17T23:13:41Z", + "cwe_ids": [ + "CWE-843" + ], + "severity": "LOW" + }, + "references": [ + { + "type": "WEB", + "url": "https://github.com/opencontainers/distribution-spec/security/advisories/GHSA-mc8v-mgrf-8f4m" + }, + { + "type": "WEB", + "url": "https://github.com/opencontainers/image-spec/security/advisories/GHSA-77vh-xpmg-72qh" + }, + { + "type": "WEB", + "url": "https://github.com/opencontainers/image-spec/commit/693428a734f5bab1a84bd2f990d92ef1111cd60c" + }, + { + "type": "PACKAGE", + "url": "https://github.com/opencontainers/image-spec" + }, + { + "type": "WEB", + "url": "https://github.com/opencontainers/image-spec/releases/tag/v1.0.2" + } + ], + "affected": [ + { + "package": { + "name": "github.com/opencontainers/image-spec", + "ecosystem": "Go", + "purl": "pkg:golang/github.com/opencontainers/image-spec" + }, + "ranges": [ + { + "type": "SEMVER", + "events": [ + { + "introduced": "0" + }, + { + "fixed": "1.0.2" + } + ] + } + ], + "database_specific": { + "source": "https://github.com/github/advisory-database/blob/main/advisories/github-reviewed/2021/11/GHSA-77vh-xpmg-72qh/GHSA-77vh-xpmg-72qh.json" + } + } + ], + "schema_version": "1.7.3", + "severity": [ + { + "type": "CVSS_V3", + "score": "CVSS:3.1/AV:N/AC:H/PR:L/UI:R/S:C/C:N/I:L/A:N" + } + ] + }, + { + "id": "GHSA-7rjr-3q55-vv33", + "summary": "Incomplete fix for Apache Log4j vulnerability", + "details": "# Impact\n\nThe fix to address [CVE-2021-44228](https://nvd.nist.gov/vuln/detail/CVE-2021-44228) in Apache Log4j 2.15.0 was incomplete in certain non-default configurations. This could allow attackers with control over Thread Context Map (MDC) input data when the logging configuration uses a non-default Pattern Layout with either a Context Lookup (for example, $${ctx:loginId}) or a Thread Context Map pattern (%X, %mdc, or %MDC) to craft malicious input data using a JNDI Lookup pattern resulting in a remote code execution (RCE) attack. \n\n## Affected packages\nOnly the \u0060org.apache.logging.log4j:log4j-core\u0060 package is directly affected by this vulnerability. The \u0060org.apache.logging.log4j:log4j-api\u0060 should be kept at the same version as the \u0060org.apache.logging.log4j:log4j-core\u0060 package to ensure compatability if in use.\n\n# Mitigation\n\nLog4j 2.16.0 fixes this issue by removing support for message lookup patterns and disabling JNDI functionality by default. This issue can be mitigated in prior releases (\u003C 2.16.0) by removing the JndiLookup class from the classpath (example: zip -q -d log4j-core-*.jar org/apache/logging/log4j/core/lookup/JndiLookup.class).\n\nLog4j 2.15.0 restricts JNDI LDAP lookups to localhost by default. Note that previous mitigations involving configuration such as to set the system property \u0060log4j2.formatMsgNoLookups\u0060 to \u0060true\u0060 do NOT mitigate this specific vulnerability.", + "aliases": [ + "CVE-2021-45046" + ], + "modified": "2025-05-09T13:13:16.169374Z", + "published": "2021-12-14T18:01:28Z", + "database_specific": { + "github_reviewed_at": "2021-12-14T17:55:00Z", + "cwe_ids": [ + "CWE-502", + "CWE-917" + ], + "github_reviewed": true, + "severity": "CRITICAL", + "nvd_published_at": "2021-12-14T19:15:00Z" + }, + "references": [ + { + "type": "ADVISORY", + "url": "https://nvd.nist.gov/vuln/detail/CVE-2021-45046" + }, + { + "type": "WEB", + "url": "https://www.oracle.com/security-alerts/cpujul2022.html" + }, + { + "type": "WEB", + "url": "https://www.oracle.com/security-alerts/cpujan2022.html" + }, + { + "type": "WEB", + "url": "https://www.oracle.com/security-alerts/cpuapr2022.html" + }, + { + "type": "WEB", + "url": "https://www.oracle.com/security-alerts/alert-cve-2021-44228.html" + }, + { + "type": "WEB", + "url": "https://www.openwall.com/lists/oss-security/2021/12/14/4" + }, + { + "type": "WEB", + "url": "https://www.kb.cert.org/vuls/id/930724" + }, + { + "type": "WEB", + "url": "https://www.intel.com/content/www/us/en/security-center/advisory/intel-sa-00646.html" + }, + { + "type": "WEB", + "url": "https://www.debian.org/security/2021/dsa-5022" + }, + { + "type": "WEB", + "url": "https://www.cve.org/CVERecord?id=CVE-2021-44228" + }, + { + "type": "WEB", + "url": "https://security.gentoo.org/glsa/202310-16" + }, + { + "type": "WEB", + "url": "https://sec.cloudapps.cisco.com/security/center/content/CiscoSecurityAdvisory/cisco-sa-apache-log4j-qRuKNEbd" + }, + { + "type": "WEB", + "url": "https://psirt.global.sonicwall.com/vuln-detail/SNWLID-2021-0032" + }, + { + "type": "WEB", + "url": "https://logging.apache.org/log4j/2.x/security.html" + }, + { + "type": "WEB", + "url": "https://lists.fedoraproject.org/archives/list/package-announce@lists.fedoraproject.org/message/SIG7FZULMNK2XF6FZRU4VWYDQXNMUGAJ" + }, + { + "type": "WEB", + "url": "https://lists.fedoraproject.org/archives/list/package-announce@lists.fedoraproject.org/message/EOKPQGV24RRBBI4TBZUDQMM4MEH7MXCY" + }, + { + "type": "ADVISORY", + "url": "https://github.com/advisories/GHSA-jfh8-c2jp-5v3q" + }, + { + "type": "WEB", + "url": "https://cert-portal.siemens.com/productcert/pdf/ssa-714170.pdf" + }, + { + "type": "WEB", + "url": "https://cert-portal.siemens.com/productcert/pdf/ssa-661247.pdf" + }, + { + "type": "WEB", + "url": "https://cert-portal.siemens.com/productcert/pdf/ssa-479842.pdf" + }, + { + "type": "WEB", + "url": "https://cert-portal.siemens.com/productcert/pdf/ssa-397453.pdf" + }, + { + "type": "WEB", + "url": "http://www.openwall.com/lists/oss-security/2021/12/14/4" + }, + { + "type": "WEB", + "url": "http://www.openwall.com/lists/oss-security/2021/12/15/3" + }, + { + "type": "WEB", + "url": "http://www.openwall.com/lists/oss-security/2021/12/18/1" + } + ], + "affected": [ + { + "package": { + "name": "org.apache.logging.log4j:log4j-core", + "ecosystem": "Maven", + "purl": "pkg:maven/org.apache.logging.log4j/log4j-core" + }, + "ranges": [ + { + "type": "ECOSYSTEM", + "events": [ + { + "introduced": "2.13.0" + }, + { + "fixed": "2.16.0" + } + ] + } + ], + "versions": [ + "2.13.0", + "2.13.1", + "2.13.2", + "2.13.3", + "2.14.0", + "2.14.1", + "2.15.0" + ], + "database_specific": { + "source": "https://github.com/github/advisory-database/blob/main/advisories/github-reviewed/2021/12/GHSA-7rjr-3q55-vv33/GHSA-7rjr-3q55-vv33.json" + } + }, + { + "package": { + "name": "org.apache.logging.log4j:log4j-core", + "ecosystem": "Maven", + "purl": "pkg:maven/org.apache.logging.log4j/log4j-core" + }, + "ranges": [ + { + "type": "ECOSYSTEM", + "events": [ + { + "introduced": "0" + }, + { + "fixed": "2.12.2" + } + ] + } + ], + "versions": [ + "2.0", + "2.0-alpha1", + "2.0-alpha2", + "2.0-beta1", + "2.0-beta2", + "2.0-beta3", + "2.0-beta4", + "2.0-beta5", + "2.0-beta6", + "2.0-beta7", + "2.0-beta8", + "2.0-beta9", + "2.0-rc1", + "2.0-rc2", + "2.0.1", + "2.0.2", + "2.1", + "2.10.0", + "2.11.0", + "2.11.1", + "2.11.2", + "2.12.0", + "2.12.1", + "2.2", + "2.3", + "2.3.1", + "2.3.2", + "2.4", + "2.4.1", + "2.5", + "2.6", + "2.6.1", + "2.6.2", + "2.7", + "2.8", + "2.8.1", + "2.8.2", + "2.9.0", + "2.9.1" + ], + "database_specific": { + "source": "https://github.com/github/advisory-database/blob/main/advisories/github-reviewed/2021/12/GHSA-7rjr-3q55-vv33/GHSA-7rjr-3q55-vv33.json" + } + }, + { + "package": { + "name": "org.ops4j.pax.logging:pax-logging-log4j2", + "ecosystem": "Maven", + "purl": "pkg:maven/org.ops4j.pax.logging/pax-logging-log4j2" + }, + "ranges": [ + { + "type": "ECOSYSTEM", + "events": [ + { + "introduced": "1.8.0" + }, + { + "fixed": "1.9.2" + } + ] + } + ], + "versions": [ + "1.8.0", + "1.8.1", + "1.8.2", + "1.8.3", + "1.8.4", + "1.8.5", + "1.8.6", + "1.8.7", + "1.9.0", + "1.9.1" + ], + "database_specific": { + "source": "https://github.com/github/advisory-database/blob/main/advisories/github-reviewed/2021/12/GHSA-7rjr-3q55-vv33/GHSA-7rjr-3q55-vv33.json" + } + }, + { + "package": { + "name": "org.ops4j.pax.logging:pax-logging-log4j2", + "ecosystem": "Maven", + "purl": "pkg:maven/org.ops4j.pax.logging/pax-logging-log4j2" + }, + "ranges": [ + { + "type": "ECOSYSTEM", + "events": [ + { + "introduced": "1.10.0" + }, + { + "fixed": "1.10.8" + } + ] + } + ], + "versions": [ + "1.10.0", + "1.10.1", + "1.10.2", + "1.10.3", + "1.10.4", + "1.10.5", + "1.10.6", + "1.10.7" + ], + "database_specific": { + "source": "https://github.com/github/advisory-database/blob/main/advisories/github-reviewed/2021/12/GHSA-7rjr-3q55-vv33/GHSA-7rjr-3q55-vv33.json" + } + }, + { + "package": { + "name": "org.ops4j.pax.logging:pax-logging-log4j2", + "ecosystem": "Maven", + "purl": "pkg:maven/org.ops4j.pax.logging/pax-logging-log4j2" + }, + "ranges": [ + { + "type": "ECOSYSTEM", + "events": [ + { + "introduced": "1.11.0" + }, + { + "fixed": "1.11.11" + } + ] + } + ], + "versions": [ + "1.11.0", + "1.11.1", + "1.11.10", + "1.11.2", + "1.11.3", + "1.11.4", + "1.11.5", + "1.11.6", + "1.11.7", + "1.11.8", + "1.11.9" + ], + "database_specific": { + "source": "https://github.com/github/advisory-database/blob/main/advisories/github-reviewed/2021/12/GHSA-7rjr-3q55-vv33/GHSA-7rjr-3q55-vv33.json" + } + }, + { + "package": { + "name": "org.ops4j.pax.logging:pax-logging-log4j2", + "ecosystem": "Maven", + "purl": "pkg:maven/org.ops4j.pax.logging/pax-logging-log4j2" + }, + "ranges": [ + { + "type": "ECOSYSTEM", + "events": [ + { + "introduced": "2.0.0" + }, + { + "fixed": "2.0.12" + } + ] + } + ], + "versions": [ + "2.0.0", + "2.0.1", + "2.0.10", + "2.0.11", + "2.0.2", + "2.0.3", + "2.0.4", + "2.0.5", + "2.0.6", + "2.0.7", + "2.0.8", + "2.0.9" + ], + "database_specific": { + "source": "https://github.com/github/advisory-database/blob/main/advisories/github-reviewed/2021/12/GHSA-7rjr-3q55-vv33/GHSA-7rjr-3q55-vv33.json" + } + } + ], + "schema_version": "1.7.3", + "severity": [ + { + "type": "CVSS_V3", + "score": "CVSS:3.1/AV:N/AC:H/PR:N/UI:N/S:C/C:H/I:H/A:H" + } + ] + } +] \ No newline at end of file diff --git a/src/StellaOps.Feedser.Source.Osv.Tests/Fixtures/osv-npm.snapshot.json b/src/StellaOps.Feedser.Source.Osv.Tests/Fixtures/osv-npm.snapshot.json new file mode 100644 index 00000000..e3af64c8 --- /dev/null +++ b/src/StellaOps.Feedser.Source.Osv.Tests/Fixtures/osv-npm.snapshot.json @@ -0,0 +1,123 @@ +{ + "advisoryKey": "OSV-2025-npm-0001", + "affectedPackages": [ + { + "identifier": "pkg:npm/%40scope%2Fleft-pad", + "platform": "npm", + "provenance": [ + { + "fieldMask": [], + "kind": "affected", + "recordedAt": "2025-01-08T06:30:00+00:00", + "source": "osv", + "value": "pkg:npm/%40scope%2Fleft-pad" + } + ], + "statuses": [], + "type": "semver", + "versionRanges": [ + { + "fixedVersion": "2.0.0", + "introducedVersion": "0", + "lastAffectedVersion": null, + "primitives": { + "evr": null, + "hasVendorExtensions": false, + "nevra": null, + "semVer": { + "constraintExpression": null, + "fixed": "2.0.0", + "fixedInclusive": false, + "introduced": "0", + "introducedInclusive": true, + "lastAffected": null, + "lastAffectedInclusive": true + }, + "vendorExtensions": null + }, + "provenance": { + "fieldMask": [], + "kind": "range", + "recordedAt": "2025-01-08T06:30:00+00:00", + "source": "osv", + "value": "pkg:npm/%40scope%2Fleft-pad" + }, + "rangeExpression": null, + "rangeKind": "semver" + } + ] + } + ], + "aliases": [ + "CVE-2025-113", + "GHSA-3abc-3def-3ghi", + "OSV-2025-npm-0001", + "OSV-RELATED-npm-42" + ], + "cvssMetrics": [ + { + "baseScore": 9.8, + "baseSeverity": "critical", + "provenance": { + "fieldMask": [], + "kind": "cvss", + "recordedAt": "2025-01-08T06:30:00+00:00", + "source": "osv", + "value": "CVSS_V3" + }, + "vector": "CVSS:3.1/AV:N/AC:L/PR:N/UI:N/S:U/C:H/I:H/A:H", + "version": "3.1" + } + ], + "exploitKnown": false, + "language": "en", + "modified": "2025-01-08T06:30:00+00:00", + "provenance": [ + { + "fieldMask": [], + "kind": "document", + "recordedAt": "2025-01-08T07:00:00+00:00", + "source": "osv", + "value": "https://osv.dev/vulnerability/OSV-2025-npm-0001" + }, + { + "fieldMask": [], + "kind": "mapping", + "recordedAt": "2025-01-08T06:30:00+00:00", + "source": "osv", + "value": "OSV-2025-npm-0001" + } + ], + "published": "2025-01-05T12:00:00+00:00", + "references": [ + { + "kind": "advisory", + "provenance": { + "fieldMask": [], + "kind": "reference", + "recordedAt": "2025-01-08T06:30:00+00:00", + "source": "osv", + "value": "https://example.com/npm/advisory" + }, + "sourceTag": "ADVISORY", + "summary": null, + "url": "https://example.com/npm/advisory" + }, + { + "kind": "patch", + "provenance": { + "fieldMask": [], + "kind": "reference", + "recordedAt": "2025-01-08T06:30:00+00:00", + "source": "osv", + "value": "https://example.com/npm/fix" + }, + "sourceTag": "FIX", + "summary": null, + "url": "https://example.com/npm/fix" + } + ], + "severity": "critical", + "summary": "Detailed description for npm package @scope/left-pad.", + "title": "npm package vulnerability" +} diff --git a/src/StellaOps.Feedser.Source.Osv.Tests/Fixtures/osv-pypi.snapshot.json b/src/StellaOps.Feedser.Source.Osv.Tests/Fixtures/osv-pypi.snapshot.json new file mode 100644 index 00000000..7f73da07 --- /dev/null +++ b/src/StellaOps.Feedser.Source.Osv.Tests/Fixtures/osv-pypi.snapshot.json @@ -0,0 +1,123 @@ +{ + "advisoryKey": "OSV-2025-PyPI-0001", + "affectedPackages": [ + { + "identifier": "pkg:pypi/requests", + "platform": "PyPI", + "provenance": [ + { + "fieldMask": [], + "kind": "affected", + "recordedAt": "2025-01-08T06:30:00+00:00", + "source": "osv", + "value": "pkg:pypi/requests" + } + ], + "statuses": [], + "type": "semver", + "versionRanges": [ + { + "fixedVersion": "2.0.0", + "introducedVersion": "0", + "lastAffectedVersion": null, + "primitives": { + "evr": null, + "hasVendorExtensions": false, + "nevra": null, + "semVer": { + "constraintExpression": null, + "fixed": "2.0.0", + "fixedInclusive": false, + "introduced": "0", + "introducedInclusive": true, + "lastAffected": null, + "lastAffectedInclusive": true + }, + "vendorExtensions": null + }, + "provenance": { + "fieldMask": [], + "kind": "range", + "recordedAt": "2025-01-08T06:30:00+00:00", + "source": "osv", + "value": "pkg:pypi/requests" + }, + "rangeExpression": null, + "rangeKind": "semver" + } + ] + } + ], + "aliases": [ + "CVE-2025-114", + "GHSA-4abc-4def-4ghi", + "OSV-2025-PyPI-0001", + "OSV-RELATED-PyPI-42" + ], + "cvssMetrics": [ + { + "baseScore": 9.8, + "baseSeverity": "critical", + "provenance": { + "fieldMask": [], + "kind": "cvss", + "recordedAt": "2025-01-08T06:30:00+00:00", + "source": "osv", + "value": "CVSS_V3" + }, + "vector": "CVSS:3.1/AV:N/AC:L/PR:N/UI:N/S:U/C:H/I:H/A:H", + "version": "3.1" + } + ], + "exploitKnown": false, + "language": "en", + "modified": "2025-01-08T06:30:00+00:00", + "provenance": [ + { + "fieldMask": [], + "kind": "document", + "recordedAt": "2025-01-08T07:00:00+00:00", + "source": "osv", + "value": "https://osv.dev/vulnerability/OSV-2025-PyPI-0001" + }, + { + "fieldMask": [], + "kind": "mapping", + "recordedAt": "2025-01-08T06:30:00+00:00", + "source": "osv", + "value": "OSV-2025-PyPI-0001" + } + ], + "published": "2025-01-05T12:00:00+00:00", + "references": [ + { + "kind": "advisory", + "provenance": { + "fieldMask": [], + "kind": "reference", + "recordedAt": "2025-01-08T06:30:00+00:00", + "source": "osv", + "value": "https://example.com/PyPI/advisory" + }, + "sourceTag": "ADVISORY", + "summary": null, + "url": "https://example.com/PyPI/advisory" + }, + { + "kind": "patch", + "provenance": { + "fieldMask": [], + "kind": "reference", + "recordedAt": "2025-01-08T06:30:00+00:00", + "source": "osv", + "value": "https://example.com/PyPI/fix" + }, + "sourceTag": "FIX", + "summary": null, + "url": "https://example.com/PyPI/fix" + } + ], + "severity": "critical", + "summary": "Detailed description for PyPI package requests.", + "title": "PyPI package vulnerability" +} diff --git a/src/StellaOps.Feedser.Source.Osv.Tests/Osv/OsvGhsaParityRegressionTests.cs b/src/StellaOps.Feedser.Source.Osv.Tests/Osv/OsvGhsaParityRegressionTests.cs new file mode 100644 index 00000000..b47ae735 --- /dev/null +++ b/src/StellaOps.Feedser.Source.Osv.Tests/Osv/OsvGhsaParityRegressionTests.cs @@ -0,0 +1,572 @@ +using System; +using System.Collections.Generic; +using System.Diagnostics.Metrics; +using System.Globalization; +using System.IO; +using System.Linq; +using System.Net.Http; +using System.Net.Http.Headers; +using System.Security.Cryptography; +using System.Text.Json; +using System.Text.RegularExpressions; +using MongoDB.Bson; +using StellaOps.Feedser.Models; +using StellaOps.Feedser.Source.Common; +using StellaOps.Feedser.Source.Osv; +using StellaOps.Feedser.Source.Osv.Internal; +using StellaOps.Feedser.Storage.Mongo.Documents; +using StellaOps.Feedser.Storage.Mongo.Dtos; +using Xunit; + +namespace StellaOps.Feedser.Source.Osv.Tests; + +public sealed class OsvGhsaParityRegressionTests +{ + private static readonly JsonSerializerOptions SerializerOptions = new(JsonSerializerDefaults.Web); + + // Curated GHSA identifiers spanning multiple ecosystems (PyPI, npm/go, Maven) for parity coverage. + private static readonly string[] GhsaIds = + { + "GHSA-wv4w-6qv2-qqfg", // PyPI – social-auth-app-django + "GHSA-cjjf-27cc-pvmv", // PyPI – pyload-ng + "GHSA-77vh-xpmg-72qh", // Go – opencontainers/image-spec + "GHSA-7rjr-3q55-vv33" // Maven – log4j-core / pax-logging + }; + + [Fact] + public void FixtureParity_NoIssues_EmitsMetrics() + { + RegenerateFixturesIfRequested(); + + var osvAdvisories = LoadOsvAdvisories(); + var ghsaAdvisories = LoadGhsaAdvisories(); + + if (File.Exists(RebuildSentinelPath)) + { + WriteFixture("osv-ghsa.osv.json", osvAdvisories); + WriteFixture("osv-ghsa.ghsa.json", ghsaAdvisories); + File.Delete(RebuildSentinelPath); + } + + AssertSnapshot("osv-ghsa.osv.json", osvAdvisories); + AssertSnapshot("osv-ghsa.ghsa.json", ghsaAdvisories); + + var measurements = new List(); + using var listener = CreateListener(measurements); + + var report = OsvGhsaParityInspector.Compare(osvAdvisories, ghsaAdvisories); + + if (report.HasIssues) + { + foreach (var issue in report.Issues) + { + Console.WriteLine($"[Parity] Issue: {issue.GhsaId} {issue.IssueKind} {issue.Detail}"); + } + } + + Assert.False(report.HasIssues); + Assert.Equal(GhsaIds.Length, report.TotalGhsaIds); + + OsvGhsaParityDiagnostics.RecordReport(report, "fixtures"); + listener.Dispose(); + + var total = Assert.Single(measurements, entry => string.Equals(entry.Instrument, "feedser.osv_ghsa.total", StringComparison.Ordinal)); + Assert.Equal(GhsaIds.Length, total.Value); + Assert.Equal("fixtures", Assert.IsType(total.Tags["dataset"])); + + Assert.DoesNotContain(measurements, entry => string.Equals(entry.Instrument, "feedser.osv_ghsa.issues", StringComparison.Ordinal)); + } + + private static MeterListener CreateListener(List buffer) + { + var listener = new MeterListener + { + InstrumentPublished = (instrument, l) => + { + if (instrument.Meter.Name.StartsWith("StellaOps.Feedser.Models.OsvGhsaParity", StringComparison.Ordinal)) + { + l.EnableMeasurementEvents(instrument); + } + } + }; + + listener.SetMeasurementEventCallback((instrument, measurement, tags, state) => + { + var dict = new Dictionary(StringComparer.OrdinalIgnoreCase); + foreach (var tag in tags) + { + dict[tag.Key] = tag.Value; + } + + buffer.Add(new MeasurementRecord(instrument.Name, measurement, dict)); + }); + + listener.Start(); + return listener; + } + + private static IReadOnlyList LoadOsvAdvisories() + { + var path = ResolveFixturePath("osv-ghsa.raw-osv.json"); + using var document = JsonDocument.Parse(File.ReadAllText(path)); + var advisories = new List(); + foreach (var element in document.RootElement.EnumerateArray()) + { + advisories.Add(MapOsvAdvisory(element.GetRawText())); + } + advisories.Sort((a, b) => string.Compare(a.AdvisoryKey, b.AdvisoryKey, StringComparison.OrdinalIgnoreCase)); + return advisories; + } + + private static IReadOnlyList LoadGhsaAdvisories() + { + var path = ResolveFixturePath("osv-ghsa.raw-ghsa.json"); + using var document = JsonDocument.Parse(File.ReadAllText(path)); + var advisories = new List(); + foreach (var element in document.RootElement.EnumerateArray()) + { + advisories.Add(MapGhsaAdvisory(element.GetRawText())); + } + advisories.Sort((a, b) => string.Compare(a.AdvisoryKey, b.AdvisoryKey, StringComparison.OrdinalIgnoreCase)); + return advisories; + } + + private static void RegenerateFixturesIfRequested() + { + var flag = Environment.GetEnvironmentVariable("UPDATE_PARITY_FIXTURES"); + Console.WriteLine($"[Parity] UPDATE_PARITY_FIXTURES={flag ?? "(null)"}"); + + var rawOsvPath = ResolveFixturePath("osv-ghsa.raw-osv.json"); + var rawGhsaPath = ResolveFixturePath("osv-ghsa.raw-ghsa.json"); + var shouldBootstrap = !File.Exists(rawOsvPath) || !File.Exists(rawGhsaPath); + + if (!string.Equals(flag, "1", StringComparison.Ordinal) && !shouldBootstrap) + { + return; + } + + // regeneration trigger + Console.WriteLine(shouldBootstrap + ? $"[Parity] Raw fixtures missing – regenerating OSV/GHSA snapshots for {GhsaIds.Length} advisories." + : $"[Parity] Regenerating OSV/GHSA fixtures for {GhsaIds.Length} advisories."); + + using var client = new HttpClient(); + client.DefaultRequestHeaders.UserAgent.Add(new ProductInfoHeaderValue("StellaOpsParityFixtures", "1.0")); + + var osvAdvisories = new List(GhsaIds.Length); + var ghsaAdvisories = new List(GhsaIds.Length); + var rawOsv = new List(GhsaIds.Length); + var rawGhsa = new List(GhsaIds.Length); + + foreach (var ghsaId in GhsaIds) + { + var osvJson = FetchJson(client, $"https://api.osv.dev/v1/vulns/{ghsaId}"); + var ghsaJson = FetchJson(client, $"https://api.github.com/advisories/{ghsaId}"); + + using (var osvDocument = JsonDocument.Parse(osvJson)) + { + rawOsv.Add(osvDocument.RootElement.Clone()); + } + using (var ghsaDocument = JsonDocument.Parse(ghsaJson)) + { + rawGhsa.Add(ghsaDocument.RootElement.Clone()); + } + + var osv = MapOsvAdvisory(osvJson); + var ghsa = MapGhsaAdvisory(ghsaJson); + + osvAdvisories.Add(osv); + ghsaAdvisories.Add(ghsa); + } + + osvAdvisories.Sort((a, b) => string.Compare(a.AdvisoryKey, b.AdvisoryKey, StringComparison.OrdinalIgnoreCase)); + ghsaAdvisories.Sort((a, b) => string.Compare(a.AdvisoryKey, b.AdvisoryKey, StringComparison.OrdinalIgnoreCase)); + + WriteRawFixture("osv-ghsa.raw-osv.json", rawOsv); + WriteRawFixture("osv-ghsa.raw-ghsa.json", rawGhsa); + WriteFixture("osv-ghsa.osv.json", osvAdvisories); + WriteFixture("osv-ghsa.ghsa.json", ghsaAdvisories); + } + + private static string FetchJson(HttpClient client, string uri) + { + try + { + return client.GetStringAsync(uri).GetAwaiter().GetResult(); + } + catch (Exception ex) + { + throw new InvalidOperationException($"Failed to download '{uri}'.", ex); + } + } + + private static Advisory MapOsvAdvisory(string json) + { + var dto = JsonSerializer.Deserialize(json, SerializerOptions) + ?? throw new InvalidOperationException("Unable to deserialize OSV payload."); + + var documentId = Guid.NewGuid(); + var identifier = dto.Id ?? throw new InvalidOperationException("OSV payload missing id."); + var ecosystem = dto.Affected?.FirstOrDefault()?.Package?.Ecosystem ?? "osv"; + var fetchedAt = dto.Published ?? dto.Modified ?? DateTimeOffset.UtcNow; + var sha = ComputeSha256Hex(json); + + var document = new DocumentRecord( + documentId, + OsvConnectorPlugin.SourceName, + $"https://osv.dev/vulnerability/{identifier}", + fetchedAt, + sha, + DocumentStatuses.PendingMap, + "application/json", + null, + new Dictionary(StringComparer.Ordinal) { ["osv.ecosystem"] = ecosystem }, + null, + dto.Modified, + null); + + var payload = BsonDocument.Parse(json); + var dtoRecord = new DtoRecord(Guid.NewGuid(), document.Id, OsvConnectorPlugin.SourceName, "osv.v1", payload, DateTimeOffset.UtcNow); + + return OsvMapper.Map(dto, document, dtoRecord, ecosystem); + } + + private static Advisory MapGhsaAdvisory(string json) + { + using var document = JsonDocument.Parse(json); + var root = document.RootElement; + + var ghsaId = GetString(root, "ghsa_id"); + if (string.IsNullOrWhiteSpace(ghsaId)) + { + throw new InvalidOperationException("GHSA payload missing ghsa_id."); + } + + var summary = GetString(root, "summary"); + var description = GetString(root, "description"); + var severity = GetString(root, "severity")?.ToLowerInvariant(); + var published = GetDateTime(root, "published_at"); + var updated = GetDateTime(root, "updated_at"); + var recordedAt = updated ?? DateTimeOffset.UtcNow; + + var aliases = new HashSet(StringComparer.OrdinalIgnoreCase) { ghsaId }; + if (root.TryGetProperty("identifiers", out var identifiers) && identifiers.ValueKind == JsonValueKind.Array) + { + foreach (var identifier in identifiers.EnumerateArray()) + { + var value = identifier.TryGetProperty("value", out var valueElem) ? valueElem.GetString() : null; + if (!string.IsNullOrWhiteSpace(value)) + { + aliases.Add(value); + } + } + } + + var references = new List(); + if (root.TryGetProperty("references", out var referencesElem) && referencesElem.ValueKind == JsonValueKind.Array) + { + foreach (var referenceElem in referencesElem.EnumerateArray()) + { + var url = referenceElem.GetString(); + if (string.IsNullOrWhiteSpace(url)) + { + continue; + } + + var provenance = new AdvisoryProvenance("ghsa", "reference", url, recordedAt, new[] { ProvenanceFieldMasks.References }); + references.Add(new AdvisoryReference(url, DetermineReferenceKind(url), DetermineSourceTag(url), null, provenance)); + } + } + + references = references + .DistinctBy(reference => reference.Url, StringComparer.OrdinalIgnoreCase) + .OrderBy(reference => reference.Url, StringComparer.Ordinal) + .ToList(); + + var affectedPackages = BuildGhsaPackages(root, recordedAt); + var cvssMetrics = BuildGhsaCvss(root, recordedAt); + + var advisoryProvenance = new AdvisoryProvenance("ghsa", "map", ghsaId, recordedAt, new[] { ProvenanceFieldMasks.Advisory }); + + return new Advisory( + ghsaId, + string.IsNullOrWhiteSpace(summary) ? ghsaId : summary!, + string.IsNullOrWhiteSpace(description) ? summary : description, + language: "en", + published, + updated, + severity, + exploitKnown: false, + aliases, + references, + affectedPackages, + cvssMetrics, + new[] { advisoryProvenance }); + } + + private static IReadOnlyList BuildGhsaPackages(JsonElement root, DateTimeOffset recordedAt) + { + if (!root.TryGetProperty("vulnerabilities", out var vulnerabilitiesElem) || vulnerabilitiesElem.ValueKind != JsonValueKind.Array) + { + return Array.Empty(); + } + + var packages = new List(); + foreach (var entry in vulnerabilitiesElem.EnumerateArray()) + { + if (!entry.TryGetProperty("package", out var packageElem) || packageElem.ValueKind != JsonValueKind.Object) + { + continue; + } + + var ecosystem = GetString(packageElem, "ecosystem"); + var name = GetString(packageElem, "name"); + if (string.IsNullOrWhiteSpace(name)) + { + continue; + } + + var identifier = BuildIdentifier(ecosystem, name); + var packageProvenance = new AdvisoryProvenance("ghsa", "package", identifier, recordedAt, new[] { ProvenanceFieldMasks.AffectedPackages }); + + var rangeExpression = GetString(entry, "vulnerable_version_range"); + string? firstPatched = null; + if (entry.TryGetProperty("first_patched_version", out var firstPatchedElem) && firstPatchedElem.ValueKind == JsonValueKind.Object) + { + firstPatched = GetString(firstPatchedElem, "identifier"); + } + + var ranges = ParseVersionRanges(rangeExpression, firstPatched, identifier, recordedAt); + + packages.Add(new AffectedPackage( + AffectedPackageTypes.SemVer, + identifier, + ecosystem, + ranges, + Array.Empty(), + new[] { packageProvenance })); + } + + return packages.OrderBy(package => package.Identifier, StringComparer.Ordinal).ToArray(); + } + + private static IReadOnlyList ParseVersionRanges(string? vulnerableVersionRange, string? firstPatchedVersion, string identifier, DateTimeOffset recordedAt) + { + if (string.IsNullOrWhiteSpace(vulnerableVersionRange) && string.IsNullOrWhiteSpace(firstPatchedVersion)) + { + return Array.Empty(); + } + + var ranges = new List(); + + var expressions = vulnerableVersionRange? + .Split(',', StringSplitOptions.TrimEntries | StringSplitOptions.RemoveEmptyEntries) + .ToArray() ?? Array.Empty(); + + string? introduced = null; + string? fixedVersion = firstPatchedVersion; + string? lastAffected = null; + + foreach (var expression in expressions) + { + if (expression.StartsWith(">=", StringComparison.Ordinal)) + { + introduced = expression[(expression.IndexOf('=') + 1)..].Trim(); + } + else if (expression.StartsWith(">", StringComparison.Ordinal)) + { + introduced = expression[1..].Trim(); + } + else if (expression.StartsWith("<=", StringComparison.Ordinal)) + { + lastAffected = expression[(expression.IndexOf('=') + 1)..].Trim(); + } + else if (expression.StartsWith("<", StringComparison.Ordinal)) + { + fixedVersion = expression[1..].Trim(); + } + } + + var provenance = new AdvisoryProvenance("ghsa", "range", identifier, recordedAt, new[] { ProvenanceFieldMasks.VersionRanges }); + ranges.Add(new AffectedVersionRange("semver", NullIfWhitespace(introduced), NullIfWhitespace(fixedVersion), NullIfWhitespace(lastAffected), vulnerableVersionRange, provenance)); + + return ranges; + } + + private static IReadOnlyList BuildGhsaCvss(JsonElement root, DateTimeOffset recordedAt) + { + if (!root.TryGetProperty("cvss_severities", out var severitiesElem) || severitiesElem.ValueKind != JsonValueKind.Object) + { + return Array.Empty(); + } + + var metrics = new List(); + if (severitiesElem.TryGetProperty("cvss_v3", out var cvssElem) && cvssElem.ValueKind == JsonValueKind.Object) + { + var vector = GetString(cvssElem, "vector_string"); + if (!string.IsNullOrWhiteSpace(vector)) + { + var score = cvssElem.TryGetProperty("score", out var scoreElem) && scoreElem.ValueKind == JsonValueKind.Number + ? scoreElem.GetDouble() + : 0d; + var provenance = new AdvisoryProvenance("ghsa", "cvss", vector, recordedAt, new[] { ProvenanceFieldMasks.CvssMetrics }); + var version = vector.StartsWith("CVSS:4.0", StringComparison.OrdinalIgnoreCase) ? "4.0" : "3.1"; + var severity = GetString(root, "severity")?.ToLowerInvariant() ?? "unknown"; + metrics.Add(new CvssMetric(version, vector, score, severity, provenance)); + } + } + + return metrics; + } + + private static string BuildIdentifier(string? ecosystem, string name) + { + if (string.IsNullOrWhiteSpace(ecosystem)) + { + return name; + } + + var key = ecosystem.Trim().ToLowerInvariant(); + return key switch + { + "pypi" => $"pkg:pypi/{name.Replace('_', '-').ToLowerInvariant()}", + "npm" => $"pkg:npm/{name.ToLowerInvariant()}", + "maven" => $"pkg:maven/{name.Replace(':', '/')}", + "go" or "golang" => $"pkg:golang/{name}", + _ => name + }; + } + + private static string? DetermineReferenceKind(string url) + { + if (url.Contains("/commit/", StringComparison.OrdinalIgnoreCase) || + url.Contains("/pull/", StringComparison.OrdinalIgnoreCase) || + url.Contains("/releases/tag/", StringComparison.OrdinalIgnoreCase) || + url.Contains("/pull-requests/", StringComparison.OrdinalIgnoreCase)) + { + return "patch"; + } + + if (url.Contains("advisories", StringComparison.OrdinalIgnoreCase) || + url.Contains("security", StringComparison.OrdinalIgnoreCase) || + url.Contains("cve", StringComparison.OrdinalIgnoreCase)) + { + return "advisory"; + } + + return null; + } + + private static string? DetermineSourceTag(string url) + { + if (Uri.TryCreate(url, UriKind.Absolute, out var uri)) + { + return uri.Host; + } + + return null; + } + + private static string? GetString(JsonElement element, string propertyName) + { + if (element.TryGetProperty(propertyName, out var property)) + { + if (property.ValueKind == JsonValueKind.String) + { + return property.GetString(); + } + } + + return null; + } + + private static DateTimeOffset? GetDateTime(JsonElement element, string propertyName) + { + if (element.TryGetProperty(propertyName, out var property) && property.ValueKind == JsonValueKind.String) + { + if (property.TryGetDateTimeOffset(out var value)) + { + return value; + } + } + + return null; + } + + private static void WriteFixture(string filename, IReadOnlyList advisories) + { + var path = ResolveFixturePath(filename); + var directory = Path.GetDirectoryName(path); + if (!string.IsNullOrEmpty(directory)) + { + Directory.CreateDirectory(directory); + } + + var snapshot = SnapshotSerializer.ToSnapshot(advisories); + File.WriteAllText(path, snapshot); + } + + private static void WriteRawFixture(string filename, IReadOnlyList elements) + { + var path = ResolveFixturePath(filename); + var directory = Path.GetDirectoryName(path); + if (!string.IsNullOrEmpty(directory)) + { + Directory.CreateDirectory(directory); + } + + var json = JsonSerializer.Serialize(elements, new JsonSerializerOptions + { + WriteIndented = true + }); + File.WriteAllText(path, json); + } + + private static void AssertSnapshot(string filename, IReadOnlyList advisories) + { + var path = ResolveFixturePath(filename); + var actual = File.ReadAllText(path).Trim().ReplaceLineEndings("\n"); + var expected = SnapshotSerializer.ToSnapshot(advisories).Trim().ReplaceLineEndings("\n"); + + var normalizedActual = NormalizeRecordedAt(actual); + var normalizedExpected = NormalizeRecordedAt(expected); + + if (!string.Equals(normalizedExpected, normalizedActual, StringComparison.Ordinal)) + { + var shouldUpdate = string.Equals(Environment.GetEnvironmentVariable("UPDATE_PARITY_FIXTURES"), "1", StringComparison.Ordinal); + if (shouldUpdate) + { + var normalized = expected.Replace("\n", Environment.NewLine, StringComparison.Ordinal); + File.WriteAllText(path, normalized); + actual = expected; + normalizedActual = normalizedExpected; + } + } + + Assert.Equal(normalizedExpected, normalizedActual); + } + + private static string ResolveFixturePath(string filename) + => Path.Combine(ProjectFixtureDirectory, filename); + + private static string NormalizeRecordedAt(string input) + => RecordedAtRegex.Replace(input, "\"recordedAt\": \"#normalized#\""); + + private static string ProjectFixtureDirectory { get; } = Path.GetFullPath(Path.Combine(AppContext.BaseDirectory, "..", "..", "..", "Fixtures")); + + private static string RebuildSentinelPath => Path.Combine(ProjectFixtureDirectory, ".rebuild"); + + private static readonly Regex RecordedAtRegex = new("\"recordedAt\": \"[^\"]+\"", RegexOptions.CultureInvariant | RegexOptions.Compiled); + + private static string ComputeSha256Hex(string payload) + { + var bytes = SHA256.HashData(System.Text.Encoding.UTF8.GetBytes(payload)); + return Convert.ToHexString(bytes); + } + + private static string? NullIfWhitespace(string? value) + => string.IsNullOrWhiteSpace(value) ? null : value.Trim(); + + private sealed record MeasurementRecord(string Instrument, long Value, IReadOnlyDictionary Tags); + +} diff --git a/src/StellaOps.Feedser.Source.Osv.Tests/Osv/OsvMapperTests.cs b/src/StellaOps.Feedser.Source.Osv.Tests/Osv/OsvMapperTests.cs new file mode 100644 index 00000000..40b2b91e --- /dev/null +++ b/src/StellaOps.Feedser.Source.Osv.Tests/Osv/OsvMapperTests.cs @@ -0,0 +1,123 @@ +using System; +using System.Collections.Generic; +using System.Text.Json; +using MongoDB.Bson; +using StellaOps.Feedser.Models; +using StellaOps.Feedser.Source.Common; +using StellaOps.Feedser.Source.Osv; +using StellaOps.Feedser.Source.Osv.Internal; +using StellaOps.Feedser.Storage.Mongo.Documents; +using StellaOps.Feedser.Storage.Mongo.Dtos; +using Xunit; + +namespace StellaOps.Feedser.Source.Osv.Tests; + +public sealed class OsvMapperTests +{ + [Fact] + public void Map_NormalizesAliasesReferencesAndRanges() + { + var published = DateTimeOffset.UtcNow.AddDays(-2); + var modified = DateTimeOffset.UtcNow.AddDays(-1); + + using var databaseSpecificJson = JsonDocument.Parse("{}"); + using var ecosystemSpecificJson = JsonDocument.Parse("{}"); + + var dto = new OsvVulnerabilityDto + { + Id = "OSV-2025-TEST", + Summary = "Test summary", + Details = "Longer details for the advisory.", + Published = published, + Modified = modified, + Aliases = new[] { "CVE-2025-0001", "CVE-2025-0001", "GHSA-xxxx" }, + Related = new[] { "CVE-2025-0002" }, + References = new[] + { + new OsvReferenceDto { Url = "https://example.com/advisory", Type = "ADVISORY" }, + new OsvReferenceDto { Url = "https://example.com/advisory", Type = "ADVISORY" }, + new OsvReferenceDto { Url = "https://example.com/patch", Type = "PATCH" }, + }, + DatabaseSpecific = databaseSpecificJson.RootElement, + Severity = new[] + { + new OsvSeverityDto { Type = "CVSS_V3", Score = "CVSS:3.1/AV:N/AC:L/PR:N/UI:N/S:U/C:H/I:H/A:H" }, + }, + Affected = new[] + { + new OsvAffectedPackageDto + { + Package = new OsvPackageDto + { + Ecosystem = "PyPI", + Name = "example", + Purl = "pkg:pypi/example", + }, + Ranges = new[] + { + new OsvRangeDto + { + Type = "SEMVER", + Events = new[] + { + new OsvEventDto { Introduced = "0" }, + new OsvEventDto { Fixed = "1.0.1" }, + } + } + }, + EcosystemSpecific = ecosystemSpecificJson.RootElement, + } + } + }; + + var document = new DocumentRecord( + Guid.NewGuid(), + OsvConnectorPlugin.SourceName, + "https://osv.dev/vulnerability/OSV-2025-TEST", + DateTimeOffset.UtcNow, + "sha256", + DocumentStatuses.PendingParse, + "application/json", + null, + new Dictionary(StringComparer.Ordinal) + { + ["osv.ecosystem"] = "PyPI", + }, + null, + modified, + null, + null); + + var payload = BsonDocument.Parse(JsonSerializer.Serialize(dto, new JsonSerializerOptions(JsonSerializerDefaults.Web) + { + DefaultIgnoreCondition = System.Text.Json.Serialization.JsonIgnoreCondition.WhenWritingNull, + })); + var dtoRecord = new DtoRecord(Guid.NewGuid(), document.Id, OsvConnectorPlugin.SourceName, "osv.v1", payload, DateTimeOffset.UtcNow); + + var advisory = OsvMapper.Map(dto, document, dtoRecord, "PyPI"); + + Assert.Equal(dto.Id, advisory.AdvisoryKey); + Assert.Contains("CVE-2025-0002", advisory.Aliases); + Assert.Equal(4, advisory.Aliases.Length); + + Assert.Equal(2, advisory.References.Length); + Assert.Equal("https://example.com/advisory", advisory.References[0].Url); + Assert.Equal("https://example.com/patch", advisory.References[1].Url); + + Assert.Single(advisory.AffectedPackages); + var affected = advisory.AffectedPackages[0]; + Assert.Equal(AffectedPackageTypes.SemVer, affected.Type); + Assert.Single(affected.VersionRanges); + Assert.Equal("0", affected.VersionRanges[0].IntroducedVersion); + Assert.Equal("1.0.1", affected.VersionRanges[0].FixedVersion); + var semver = affected.VersionRanges[0].Primitives?.SemVer; + Assert.NotNull(semver); + Assert.Equal("0", semver!.Introduced); + Assert.True(semver.IntroducedInclusive); + Assert.Equal("1.0.1", semver.Fixed); + Assert.False(semver.FixedInclusive); + + Assert.Single(advisory.CvssMetrics); + Assert.Equal("3.1", advisory.CvssMetrics[0].Version); + } +} diff --git a/src/StellaOps.Feedser.Source.Osv.Tests/Osv/OsvSnapshotTests.cs b/src/StellaOps.Feedser.Source.Osv.Tests/Osv/OsvSnapshotTests.cs new file mode 100644 index 00000000..6ddfcd47 --- /dev/null +++ b/src/StellaOps.Feedser.Source.Osv.Tests/Osv/OsvSnapshotTests.cs @@ -0,0 +1,141 @@ +using System; +using System.Collections.Generic; +using System.IO; +using System.Text.Json; +using MongoDB.Bson; +using StellaOps.Feedser.Models; +using StellaOps.Feedser.Source.Osv; +using StellaOps.Feedser.Source.Osv.Internal; +using StellaOps.Feedser.Storage.Mongo.Documents; +using StellaOps.Feedser.Storage.Mongo.Dtos; +using StellaOps.Feedser.Source.Common; +using Xunit; +using Xunit.Abstractions; + +namespace StellaOps.Feedser.Source.Osv.Tests; + +public sealed class OsvSnapshotTests +{ + private static readonly DateTimeOffset BaselinePublished = new(2025, 1, 5, 12, 0, 0, TimeSpan.Zero); + private static readonly DateTimeOffset BaselineModified = new(2025, 1, 8, 6, 30, 0, TimeSpan.Zero); + private static readonly DateTimeOffset BaselineFetched = new(2025, 1, 8, 7, 0, 0, TimeSpan.Zero); + + private readonly ITestOutputHelper _output; + + public OsvSnapshotTests(ITestOutputHelper output) + { + _output = output; + } + + [Theory] + [InlineData("PyPI", "pkg:pypi/requests", "requests", "osv-pypi.snapshot.json")] + [InlineData("npm", "pkg:npm/%40scope%2Fleft-pad", "@scope/left-pad", "osv-npm.snapshot.json")] + public void Map_ProducesExpectedSnapshot(string ecosystem, string purl, string packageName, string snapshotFile) + { + var dto = CreateDto(ecosystem, purl, packageName); + var document = CreateDocumentRecord(ecosystem); + var dtoRecord = CreateDtoRecord(document, dto); + + var advisory = OsvMapper.Map(dto, document, dtoRecord, ecosystem); + var actual = SnapshotSerializer.ToSnapshot(advisory).Trim(); + + var snapshotPath = Path.Combine(AppContext.BaseDirectory, "Fixtures", snapshotFile); + var expected = File.Exists(snapshotPath) ? File.ReadAllText(snapshotPath).Trim() : string.Empty; + + if (!string.Equals(actual, expected, StringComparison.Ordinal)) + { + _output.WriteLine(actual); + } + + Assert.False(string.IsNullOrEmpty(expected), $"Snapshot '{snapshotFile}' not found or empty."); + + using var expectedJson = JsonDocument.Parse(expected); + using var actualJson = JsonDocument.Parse(actual); + Assert.True(JsonElement.DeepEquals(actualJson.RootElement, expectedJson.RootElement), "OSV snapshot mismatch."); + } + + private static OsvVulnerabilityDto CreateDto(string ecosystem, string purl, string packageName) + { + return new OsvVulnerabilityDto + { + Id = $"OSV-2025-{ecosystem}-0001", + Summary = $"{ecosystem} package vulnerability", + Details = $"Detailed description for {ecosystem} package {packageName}.", + Published = BaselinePublished, + Modified = BaselineModified, + Aliases = new[] { $"CVE-2025-11{ecosystem.Length}", $"GHSA-{ecosystem.Length}abc-{ecosystem.Length}def-{ecosystem.Length}ghi" }, + Related = new[] { $"OSV-RELATED-{ecosystem}-42" }, + References = new[] + { + new OsvReferenceDto { Url = $"https://example.com/{ecosystem}/advisory", Type = "ADVISORY" }, + new OsvReferenceDto { Url = $"https://example.com/{ecosystem}/fix", Type = "FIX" }, + }, + Severity = new[] + { + new OsvSeverityDto { Type = "CVSS_V3", Score = "CVSS:3.1/AV:N/AC:L/PR:N/UI:N/S:U/C:H/I:H/A:H" }, + }, + Affected = new[] + { + new OsvAffectedPackageDto + { + Package = new OsvPackageDto + { + Ecosystem = ecosystem, + Name = packageName, + Purl = purl, + }, + Ranges = new[] + { + new OsvRangeDto + { + Type = "SEMVER", + Events = new[] + { + new OsvEventDto { Introduced = "0" }, + new OsvEventDto { Fixed = "2.0.0" }, + } + } + }, + Versions = new[] { "1.0.0", "1.5.0" }, + EcosystemSpecific = ParseElement("{\"severity\":\"high\"}"), + } + }, + DatabaseSpecific = ParseElement("{\"source\":\"osv.dev\"}"), + }; + } + + private static DocumentRecord CreateDocumentRecord(string ecosystem) + => new( + Guid.Parse("11111111-1111-1111-1111-111111111111"), + OsvConnectorPlugin.SourceName, + $"https://osv.dev/vulnerability/OSV-2025-{ecosystem}-0001", + BaselineFetched, + "sha256-osv-snapshot", + DocumentStatuses.PendingParse, + "application/json", + null, + new Dictionary(StringComparer.Ordinal) + { + ["osv.ecosystem"] = ecosystem, + }, + "\"osv-etag\"", + BaselineModified, + null, + null); + + private static DtoRecord CreateDtoRecord(DocumentRecord document, OsvVulnerabilityDto dto) + { + var payload = BsonDocument.Parse(JsonSerializer.Serialize(dto, new JsonSerializerOptions(JsonSerializerDefaults.Web) + { + DefaultIgnoreCondition = System.Text.Json.Serialization.JsonIgnoreCondition.WhenWritingNull, + })); + + return new DtoRecord(Guid.Parse("22222222-2222-2222-2222-222222222222"), document.Id, OsvConnectorPlugin.SourceName, "osv.v1", payload, BaselineModified); + } + + private static JsonElement ParseElement(string json) + { + using var document = JsonDocument.Parse(json); + return document.RootElement.Clone(); + } +} diff --git a/src/StellaOps.Feedser.Source.Osv.Tests/StellaOps.Feedser.Source.Osv.Tests.csproj b/src/StellaOps.Feedser.Source.Osv.Tests/StellaOps.Feedser.Source.Osv.Tests.csproj new file mode 100644 index 00000000..e501ae0d --- /dev/null +++ b/src/StellaOps.Feedser.Source.Osv.Tests/StellaOps.Feedser.Source.Osv.Tests.csproj @@ -0,0 +1,18 @@ + + + net10.0 + enable + enable + + + + + + + + + + PreserveNewest + + + diff --git a/src/StellaOps.Feedser.Source.Osv/AGENTS.md b/src/StellaOps.Feedser.Source.Osv/AGENTS.md new file mode 100644 index 00000000..40f0ddf0 --- /dev/null +++ b/src/StellaOps.Feedser.Source.Osv/AGENTS.md @@ -0,0 +1,26 @@ +# AGENTS +## Role +Connector for OSV.dev across ecosystems; authoritative SemVer/PURL ranges for OSS packages. +## Scope +- Fetch by ecosystem or time range; handle pagination and changed-since cursors. +- Parse OSV JSON; validate schema; capture introduced/fixed events, database_specific where relevant. +- Map to Advisory with AffectedPackage(type=semver, Identifier=PURL); preserve SemVer constraints and introduced/fixed chronology. +- Maintain per-ecosystem cursors and deduplicate runs via payload hashes to keep reruns idempotent. +## Participants +- Source.Common supplies HTTP clients, pagination helpers, and validators. +- Storage.Mongo persists documents, DTOs, advisories, and source_state cursors. +- Merge engine resolves OSV vs GHSA consistency; prefers SemVer data for libraries; distro OVAL still overrides OS packages. +- Exporters serialize per-ecosystem ranges untouched. +## Interfaces & contracts +- Job kinds: osv:fetch, osv:parse, osv:map (naming consistent with other connectors). +- Aliases include CVE/GHSA/OSV IDs; references include advisory/patch/release URLs. +- Provenance records method=parser and source=osv. +## In/Out of scope +In: SemVer+PURL accuracy for OSS ecosystems. +Out: vendor PSIRT and distro OVAL specifics. +## Observability & security expectations +- Metrics: SourceDiagnostics exposes the shared `feedser.source.http.*` counters/histograms tagged `feedser.source=osv`; observability dashboards slice on the tag to monitor item volume, schema failures, range counts, and ecosystem coverage. Logs include ecosystem and cursor values. +## Tests +- Author and review coverage in `../StellaOps.Feedser.Source.Osv.Tests`. +- Shared fixtures (e.g., `MongoIntegrationFixture`, `ConnectorTestHarness`) live in `../StellaOps.Feedser.Testing`. +- Keep fixtures deterministic; match new cases to real-world advisories or regression scenarios. diff --git a/src/StellaOps.Feedser.Source.Osv/Configuration/OsvOptions.cs b/src/StellaOps.Feedser.Source.Osv/Configuration/OsvOptions.cs new file mode 100644 index 00000000..c6d5f333 --- /dev/null +++ b/src/StellaOps.Feedser.Source.Osv/Configuration/OsvOptions.cs @@ -0,0 +1,81 @@ +using System.Diagnostics.CodeAnalysis; + +namespace StellaOps.Feedser.Source.Osv.Configuration; + +public sealed class OsvOptions +{ + public const string HttpClientName = "source.osv"; + + public Uri BaseUri { get; set; } = new("https://osv-vulnerabilities.storage.googleapis.com/", UriKind.Absolute); + + public IReadOnlyList Ecosystems { get; set; } = new[] { "PyPI", "npm", "Maven", "Go", "crates" }; + + public TimeSpan InitialBackfill { get; set; } = TimeSpan.FromDays(14); + + public TimeSpan ModifiedTolerance { get; set; } = TimeSpan.FromMinutes(10); + + public int MaxAdvisoriesPerFetch { get; set; } = 250; + + public string ArchiveFileName { get; set; } = "all.zip"; + + public TimeSpan RequestDelay { get; set; } = TimeSpan.FromMilliseconds(250); + + public TimeSpan HttpTimeout { get; set; } = TimeSpan.FromMinutes(3); + + [MemberNotNull(nameof(BaseUri), nameof(Ecosystems), nameof(ArchiveFileName))] + public void Validate() + { + if (BaseUri is null || !BaseUri.IsAbsoluteUri) + { + throw new InvalidOperationException("OSV base URI must be an absolute URI."); + } + + if (string.IsNullOrWhiteSpace(ArchiveFileName)) + { + throw new InvalidOperationException("OSV archive file name must be provided."); + } + + if (!ArchiveFileName.EndsWith(".zip", StringComparison.OrdinalIgnoreCase)) + { + throw new InvalidOperationException("OSV archive file name must be a .zip resource."); + } + + if (Ecosystems is null || Ecosystems.Count == 0) + { + throw new InvalidOperationException("At least one OSV ecosystem must be configured."); + } + + foreach (var ecosystem in Ecosystems) + { + if (string.IsNullOrWhiteSpace(ecosystem)) + { + throw new InvalidOperationException("Ecosystem names cannot be null or whitespace."); + } + } + + if (InitialBackfill <= TimeSpan.Zero) + { + throw new InvalidOperationException("Initial backfill window must be positive."); + } + + if (ModifiedTolerance < TimeSpan.Zero) + { + throw new InvalidOperationException("Modified tolerance cannot be negative."); + } + + if (MaxAdvisoriesPerFetch <= 0) + { + throw new InvalidOperationException("Max advisories per fetch must be greater than zero."); + } + + if (RequestDelay < TimeSpan.Zero) + { + throw new InvalidOperationException("Request delay cannot be negative."); + } + + if (HttpTimeout <= TimeSpan.Zero) + { + throw new InvalidOperationException("HTTP timeout must be positive."); + } + } +} diff --git a/src/StellaOps.Feedser.Source.Osv/Internal/OsvCursor.cs b/src/StellaOps.Feedser.Source.Osv/Internal/OsvCursor.cs new file mode 100644 index 00000000..a11cf28a --- /dev/null +++ b/src/StellaOps.Feedser.Source.Osv/Internal/OsvCursor.cs @@ -0,0 +1,290 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using MongoDB.Bson; + +namespace StellaOps.Feedser.Source.Osv.Internal; + +internal sealed record OsvCursor( + IReadOnlyDictionary LastModifiedByEcosystem, + IReadOnlyDictionary> ProcessedIdsByEcosystem, + IReadOnlyDictionary ArchiveMetadataByEcosystem, + IReadOnlyCollection PendingDocuments, + IReadOnlyCollection PendingMappings) +{ + private static readonly IReadOnlyDictionary EmptyLastModified = + new Dictionary(StringComparer.OrdinalIgnoreCase); + private static readonly IReadOnlyDictionary> EmptyProcessedIds = + new Dictionary>(StringComparer.OrdinalIgnoreCase); + private static readonly IReadOnlyDictionary EmptyArchiveMetadata = + new Dictionary(StringComparer.OrdinalIgnoreCase); + private static readonly IReadOnlyCollection EmptyGuidList = Array.Empty(); + private static readonly IReadOnlyCollection EmptyStringList = Array.Empty(); + + public static OsvCursor Empty { get; } = new(EmptyLastModified, EmptyProcessedIds, EmptyArchiveMetadata, EmptyGuidList, EmptyGuidList); + + public BsonDocument ToBsonDocument() + { + var document = new BsonDocument + { + ["pendingDocuments"] = new BsonArray(PendingDocuments.Select(id => id.ToString())), + ["pendingMappings"] = new BsonArray(PendingMappings.Select(id => id.ToString())), + }; + + if (LastModifiedByEcosystem.Count > 0) + { + var lastModifiedDoc = new BsonDocument(); + foreach (var (ecosystem, timestamp) in LastModifiedByEcosystem) + { + lastModifiedDoc[ecosystem] = timestamp.HasValue ? BsonValue.Create(timestamp.Value.UtcDateTime) : BsonNull.Value; + } + + document["lastModified"] = lastModifiedDoc; + } + + if (ProcessedIdsByEcosystem.Count > 0) + { + var processedDoc = new BsonDocument(); + foreach (var (ecosystem, ids) in ProcessedIdsByEcosystem) + { + processedDoc[ecosystem] = new BsonArray(ids.Select(id => id)); + } + + document["processed"] = processedDoc; + } + + if (ArchiveMetadataByEcosystem.Count > 0) + { + var metadataDoc = new BsonDocument(); + foreach (var (ecosystem, metadata) in ArchiveMetadataByEcosystem) + { + var element = new BsonDocument(); + if (!string.IsNullOrWhiteSpace(metadata.ETag)) + { + element["etag"] = metadata.ETag; + } + + if (metadata.LastModified.HasValue) + { + element["lastModified"] = metadata.LastModified.Value.UtcDateTime; + } + + metadataDoc[ecosystem] = element; + } + + document["archive"] = metadataDoc; + } + + return document; + } + + public static OsvCursor FromBson(BsonDocument? document) + { + if (document is null || document.ElementCount == 0) + { + return Empty; + } + + var lastModified = ReadLastModified(document.TryGetValue("lastModified", out var lastModifiedValue) ? lastModifiedValue : null); + var processed = ReadProcessedIds(document.TryGetValue("processed", out var processedValue) ? processedValue : null); + var archiveMetadata = ReadArchiveMetadata(document.TryGetValue("archive", out var archiveValue) ? archiveValue : null); + var pendingDocuments = ReadGuidList(document, "pendingDocuments"); + var pendingMappings = ReadGuidList(document, "pendingMappings"); + + return new OsvCursor(lastModified, processed, archiveMetadata, pendingDocuments, pendingMappings); + } + + public DateTimeOffset? GetLastModified(string ecosystem) + { + ArgumentException.ThrowIfNullOrEmpty(ecosystem); + return LastModifiedByEcosystem.TryGetValue(ecosystem, out var value) ? value : null; + } + + public bool HasProcessedId(string ecosystem, string id) + { + ArgumentException.ThrowIfNullOrEmpty(ecosystem); + ArgumentException.ThrowIfNullOrEmpty(id); + + return ProcessedIdsByEcosystem.TryGetValue(ecosystem, out var ids) + && ids.Contains(id, StringComparer.OrdinalIgnoreCase); + } + + public OsvCursor WithLastModified(string ecosystem, DateTimeOffset timestamp, IEnumerable processedIds) + { + ArgumentException.ThrowIfNullOrEmpty(ecosystem); + + var lastModified = new Dictionary(LastModifiedByEcosystem, StringComparer.OrdinalIgnoreCase) + { + [ecosystem] = timestamp.ToUniversalTime(), + }; + + var processed = new Dictionary>(ProcessedIdsByEcosystem, StringComparer.OrdinalIgnoreCase) + { + [ecosystem] = processedIds?.Where(static id => !string.IsNullOrWhiteSpace(id)) + .Select(static id => id.Trim()) + .Distinct(StringComparer.OrdinalIgnoreCase) + .ToArray() ?? EmptyStringList, + }; + + return this with { LastModifiedByEcosystem = lastModified, ProcessedIdsByEcosystem = processed }; + } + + public OsvCursor WithPendingDocuments(IEnumerable ids) + => this with { PendingDocuments = ids?.Distinct().ToArray() ?? EmptyGuidList }; + + public OsvCursor WithPendingMappings(IEnumerable ids) + => this with { PendingMappings = ids?.Distinct().ToArray() ?? EmptyGuidList }; + + public OsvCursor AddProcessedId(string ecosystem, string id) + { + ArgumentException.ThrowIfNullOrEmpty(ecosystem); + ArgumentException.ThrowIfNullOrEmpty(id); + + var processed = new Dictionary>(ProcessedIdsByEcosystem, StringComparer.OrdinalIgnoreCase); + if (!processed.TryGetValue(ecosystem, out var ids)) + { + ids = EmptyStringList; + } + + var set = new HashSet(ids, StringComparer.OrdinalIgnoreCase) + { + id.Trim(), + }; + + processed[ecosystem] = set.ToArray(); + return this with { ProcessedIdsByEcosystem = processed }; + } + + public bool TryGetArchiveMetadata(string ecosystem, out OsvArchiveMetadata metadata) + { + ArgumentException.ThrowIfNullOrEmpty(ecosystem); + return ArchiveMetadataByEcosystem.TryGetValue(ecosystem, out metadata!); + } + + public OsvCursor WithArchiveMetadata(string ecosystem, string? etag, DateTimeOffset? lastModified) + { + ArgumentException.ThrowIfNullOrEmpty(ecosystem); + + var metadata = new Dictionary(ArchiveMetadataByEcosystem, StringComparer.OrdinalIgnoreCase) + { + [ecosystem] = new OsvArchiveMetadata(etag?.Trim(), lastModified?.ToUniversalTime()), + }; + + return this with { ArchiveMetadataByEcosystem = metadata }; + } + + private static IReadOnlyDictionary ReadLastModified(BsonValue? value) + { + if (value is not BsonDocument document) + { + return EmptyLastModified; + } + + var dictionary = new Dictionary(StringComparer.OrdinalIgnoreCase); + foreach (var element in document.Elements) + { + if (element.Value is null || element.Value.IsBsonNull) + { + dictionary[element.Name] = null; + continue; + } + + dictionary[element.Name] = ParseDate(element.Value); + } + + return dictionary; + } + + private static IReadOnlyDictionary> ReadProcessedIds(BsonValue? value) + { + if (value is not BsonDocument document) + { + return EmptyProcessedIds; + } + + var dictionary = new Dictionary>(StringComparer.OrdinalIgnoreCase); + foreach (var element in document.Elements) + { + if (element.Value is not BsonArray array) + { + continue; + } + + var ids = new List(array.Count); + foreach (var idValue in array) + { + if (idValue?.BsonType == BsonType.String) + { + var str = idValue.AsString.Trim(); + if (!string.IsNullOrWhiteSpace(str)) + { + ids.Add(str); + } + } + } + + dictionary[element.Name] = ids.Count == 0 + ? EmptyStringList + : ids.Distinct(StringComparer.OrdinalIgnoreCase).ToArray(); + } + + return dictionary; + } + + private static IReadOnlyDictionary ReadArchiveMetadata(BsonValue? value) + { + if (value is not BsonDocument document) + { + return EmptyArchiveMetadata; + } + + var dictionary = new Dictionary(StringComparer.OrdinalIgnoreCase); + foreach (var element in document.Elements) + { + if (element.Value is not BsonDocument metadataDocument) + { + continue; + } + + string? etag = metadataDocument.TryGetValue("etag", out var etagValue) && etagValue.IsString ? etagValue.AsString : null; + DateTimeOffset? lastModified = metadataDocument.TryGetValue("lastModified", out var lastModifiedValue) + ? ParseDate(lastModifiedValue) + : null; + + dictionary[element.Name] = new OsvArchiveMetadata(etag, lastModified); + } + + return dictionary.Count == 0 ? EmptyArchiveMetadata : dictionary; + } + + private static IReadOnlyCollection ReadGuidList(BsonDocument document, string field) + { + if (!document.TryGetValue(field, out var value) || value is not BsonArray array) + { + return EmptyGuidList; + } + + var list = new List(array.Count); + foreach (var element in array) + { + if (Guid.TryParse(element.ToString(), out var guid)) + { + list.Add(guid); + } + } + + return list; + } + + private static DateTimeOffset? ParseDate(BsonValue value) + { + return value.BsonType switch + { + BsonType.DateTime => DateTime.SpecifyKind(value.ToUniversalTime(), DateTimeKind.Utc), + BsonType.String when DateTimeOffset.TryParse(value.AsString, out var parsed) => parsed.ToUniversalTime(), + _ => null, + }; + } +} + +internal sealed record OsvArchiveMetadata(string? ETag, DateTimeOffset? LastModified); diff --git a/src/StellaOps.Feedser.Source.Osv/Internal/OsvMapper.cs b/src/StellaOps.Feedser.Source.Osv/Internal/OsvMapper.cs new file mode 100644 index 00000000..4b187c6c --- /dev/null +++ b/src/StellaOps.Feedser.Source.Osv/Internal/OsvMapper.cs @@ -0,0 +1,484 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text.Json; +using StellaOps.Feedser.Models; +using StellaOps.Feedser.Normalization.Cvss; +using StellaOps.Feedser.Normalization.Identifiers; +using StellaOps.Feedser.Normalization.Text; +using StellaOps.Feedser.Source.Common; +using StellaOps.Feedser.Storage.Mongo.Documents; +using StellaOps.Feedser.Storage.Mongo.Dtos; + +namespace StellaOps.Feedser.Source.Osv.Internal; + +internal static class OsvMapper +{ + private static readonly string[] SeverityOrder = { "none", "low", "medium", "high", "critical" }; + + public static Advisory Map( + OsvVulnerabilityDto dto, + DocumentRecord document, + DtoRecord dtoRecord, + string ecosystem) + { + ArgumentNullException.ThrowIfNull(dto); + ArgumentNullException.ThrowIfNull(document); + ArgumentNullException.ThrowIfNull(dtoRecord); + ArgumentException.ThrowIfNullOrEmpty(ecosystem); + + var recordedAt = dtoRecord.ValidatedAt; + var fetchProvenance = new AdvisoryProvenance( + OsvConnectorPlugin.SourceName, + "document", + document.Uri, + document.FetchedAt, + new[] { ProvenanceFieldMasks.Advisory }); + var mappingProvenance = new AdvisoryProvenance( + OsvConnectorPlugin.SourceName, + "mapping", + dto.Id, + recordedAt, + new[] { ProvenanceFieldMasks.Advisory }); + + var aliases = BuildAliases(dto); + var references = BuildReferences(dto, recordedAt); + var credits = BuildCredits(dto, recordedAt); + var affectedPackages = BuildAffectedPackages(dto, ecosystem, recordedAt); + var cvssMetrics = BuildCvssMetrics(dto, recordedAt, out var severity); + + var normalizedDescription = DescriptionNormalizer.Normalize(new[] + { + new LocalizedText(dto.Details, "en"), + new LocalizedText(dto.Summary, "en"), + }); + + var title = string.IsNullOrWhiteSpace(dto.Summary) ? dto.Id : dto.Summary!.Trim(); + var summary = string.IsNullOrWhiteSpace(normalizedDescription.Text) ? dto.Summary : normalizedDescription.Text; + var language = string.IsNullOrWhiteSpace(normalizedDescription.Language) ? null : normalizedDescription.Language; + + return new Advisory( + dto.Id, + title, + summary, + language, + dto.Published?.ToUniversalTime(), + dto.Modified?.ToUniversalTime(), + severity, + exploitKnown: false, + aliases, + credits, + references, + affectedPackages, + cvssMetrics, + new[] { fetchProvenance, mappingProvenance }); + } + + private static IEnumerable BuildAliases(OsvVulnerabilityDto dto) + { + var aliases = new HashSet(StringComparer.OrdinalIgnoreCase) + { + dto.Id, + }; + + if (dto.Aliases is not null) + { + foreach (var alias in dto.Aliases) + { + if (!string.IsNullOrWhiteSpace(alias)) + { + aliases.Add(alias.Trim()); + } + } + } + + if (dto.Related is not null) + { + foreach (var related in dto.Related) + { + if (!string.IsNullOrWhiteSpace(related)) + { + aliases.Add(related.Trim()); + } + } + } + + return aliases; + } + + private static IReadOnlyList BuildReferences(OsvVulnerabilityDto dto, DateTimeOffset recordedAt) + { + if (dto.References is null || dto.References.Count == 0) + { + return Array.Empty(); + } + + var references = new List(dto.References.Count); + foreach (var reference in dto.References) + { + if (string.IsNullOrWhiteSpace(reference.Url)) + { + continue; + } + + var kind = NormalizeReferenceKind(reference.Type); + var provenance = new AdvisoryProvenance( + OsvConnectorPlugin.SourceName, + "reference", + reference.Url, + recordedAt, + new[] { ProvenanceFieldMasks.References }); + + try + { + references.Add(new AdvisoryReference(reference.Url, kind, reference.Type, null, provenance)); + } + catch (ArgumentException) + { + // ignore invalid URLs + } + } + + if (references.Count <= 1) + { + return references; + } + + references.Sort(CompareReferences); + + var deduped = new List(references.Count); + string? lastUrl = null; + foreach (var reference in references) + { + if (lastUrl is not null && string.Equals(lastUrl, reference.Url, StringComparison.OrdinalIgnoreCase)) + { + continue; + } + + deduped.Add(reference); + lastUrl = reference.Url; + } + + return deduped; + } + + private static string? NormalizeReferenceKind(string? type) + { + if (string.IsNullOrWhiteSpace(type)) + { + return null; + } + + return type.Trim().ToLowerInvariant() switch + { + "advisory" => "advisory", + "exploit" => "exploit", + "fix" or "patch" => "patch", + "report" => "report", + "article" => "article", + _ => null, + }; + } + + private static IReadOnlyList BuildAffectedPackages(OsvVulnerabilityDto dto, string ecosystem, DateTimeOffset recordedAt) + { + if (dto.Affected is null || dto.Affected.Count == 0) + { + return Array.Empty(); + } + + var packages = new List(dto.Affected.Count); + foreach (var affected in dto.Affected) + { + if (affected.Package is null) + { + continue; + } + + var identifier = DetermineIdentifier(affected.Package, ecosystem); + if (identifier is null) + { + continue; + } + + var provenance = new[] + { + new AdvisoryProvenance( + OsvConnectorPlugin.SourceName, + "affected", + identifier, + recordedAt, + new[] { ProvenanceFieldMasks.AffectedPackages }), + }; + + var ranges = BuildVersionRanges(affected, recordedAt, identifier); + + packages.Add(new AffectedPackage( + AffectedPackageTypes.SemVer, + identifier, + platform: affected.Package.Ecosystem, + versionRanges: ranges, + statuses: Array.Empty(), + provenance: provenance)); + } + + return packages; + } + + private static IReadOnlyList BuildCredits(OsvVulnerabilityDto dto, DateTimeOffset recordedAt) + { + if (dto.Credits is null || dto.Credits.Count == 0) + { + return Array.Empty(); + } + + var credits = new List(dto.Credits.Count); + foreach (var credit in dto.Credits) + { + var displayName = Validation.TrimToNull(credit.Name); + if (displayName is null) + { + continue; + } + + var contacts = credit.Contact is null + ? Array.Empty() + : credit.Contact + .Where(static contact => !string.IsNullOrWhiteSpace(contact)) + .Select(static contact => contact.Trim()) + .Where(static contact => contact.Length > 0) + .ToArray(); + + var provenance = new AdvisoryProvenance( + OsvConnectorPlugin.SourceName, + "credit", + displayName, + recordedAt, + new[] { ProvenanceFieldMasks.Credits }); + + credits.Add(new AdvisoryCredit(displayName, credit.Type, contacts, provenance)); + } + + return credits.Count == 0 ? Array.Empty() : credits; + } + + private static IReadOnlyList BuildVersionRanges(OsvAffectedPackageDto affected, DateTimeOffset recordedAt, string identifier) + { + if (affected.Ranges is null || affected.Ranges.Count == 0) + { + return Array.Empty(); + } + + var ranges = new List(); + foreach (var range in affected.Ranges) + { + if (!"semver".Equals(range.Type, StringComparison.OrdinalIgnoreCase) + && !"ecosystem".Equals(range.Type, StringComparison.OrdinalIgnoreCase)) + { + continue; + } + + var provenance = new AdvisoryProvenance( + OsvConnectorPlugin.SourceName, + "range", + identifier, + recordedAt, + new[] { ProvenanceFieldMasks.VersionRanges }); + if (range.Events is null || range.Events.Count == 0) + { + continue; + } + + string? introduced = null; + string? lastAffected = null; + + foreach (var evt in range.Events) + { + if (!string.IsNullOrWhiteSpace(evt.Introduced)) + { + introduced = evt.Introduced.Trim(); + lastAffected = null; + } + + if (!string.IsNullOrWhiteSpace(evt.LastAffected)) + { + lastAffected = evt.LastAffected.Trim(); + } + + if (!string.IsNullOrWhiteSpace(evt.Fixed)) + { + var fixedVersion = evt.Fixed.Trim(); + ranges.Add(new AffectedVersionRange( + "semver", + introduced, + fixedVersion, + lastAffected, + rangeExpression: null, + provenance: provenance, + primitives: BuildSemVerPrimitives(introduced, fixedVersion, lastAffected))); + introduced = null; + lastAffected = null; + } + + if (!string.IsNullOrWhiteSpace(evt.Limit)) + { + lastAffected = evt.Limit.Trim(); + } + } + + if (introduced is not null || lastAffected is not null) + { + ranges.Add(new AffectedVersionRange( + "semver", + introduced, + fixedVersion: null, + lastAffected, + rangeExpression: null, + provenance: provenance, + primitives: BuildSemVerPrimitives(introduced, null, lastAffected))); + } + } + + return ranges.Count == 0 + ? Array.Empty() + : ranges; + } + + private static RangePrimitives BuildSemVerPrimitives(string? introduced, string? fixedVersion, string? lastAffected) + { + var semver = new SemVerPrimitive( + introduced, + IntroducedInclusive: true, + fixedVersion, + FixedInclusive: false, + lastAffected, + LastAffectedInclusive: true, + ConstraintExpression: null); + + return new RangePrimitives(semver, null, null, null); + } + + private static string? DetermineIdentifier(OsvPackageDto package, string ecosystem) + { + if (!string.IsNullOrWhiteSpace(package.Purl) + && IdentifierNormalizer.TryNormalizePackageUrl(package.Purl, out var normalized)) + { + return normalized; + } + + if (!string.IsNullOrWhiteSpace(package.Name)) + { + var name = package.Name.Trim(); + return string.IsNullOrWhiteSpace(package.Ecosystem) + ? $"{ecosystem}:{name}" + : $"{package.Ecosystem.Trim()}:{name}"; + } + + return null; + } + + private static IReadOnlyList BuildCvssMetrics(OsvVulnerabilityDto dto, DateTimeOffset recordedAt, out string? severity) + { + severity = null; + if (dto.Severity is null || dto.Severity.Count == 0) + { + return Array.Empty(); + } + + var metrics = new List(dto.Severity.Count); + var bestRank = -1; + + foreach (var severityEntry in dto.Severity) + { + if (string.IsNullOrWhiteSpace(severityEntry.Score)) + { + continue; + } + + if (!CvssMetricNormalizer.TryNormalize(severityEntry.Type, severityEntry.Score, null, null, out var normalized)) + { + continue; + } + + var provenance = new AdvisoryProvenance(OsvConnectorPlugin.SourceName, "cvss", severityEntry.Type ?? "osv", recordedAt); + metrics.Add(normalized.ToModel(provenance)); + + var rank = Array.IndexOf(SeverityOrder, normalized.BaseSeverity); + if (rank > bestRank) + { + bestRank = rank; + severity = normalized.BaseSeverity; + } + } + + if (bestRank < 0 && dto.DatabaseSpecific.ValueKind == JsonValueKind.Object && + dto.DatabaseSpecific.TryGetProperty("severity", out var severityProperty)) + { + var fallback = severityProperty.GetString(); + if (!string.IsNullOrWhiteSpace(fallback)) + { + severity = SeverityNormalization.Normalize(fallback); + } + } + + return metrics; + } + + private static int CompareReferences(AdvisoryReference? left, AdvisoryReference? right) + { + if (ReferenceEquals(left, right)) + { + return 0; + } + + if (left is null) + { + return 1; + } + + if (right is null) + { + return -1; + } + + var compare = StringComparer.OrdinalIgnoreCase.Compare(left.Url, right.Url); + if (compare != 0) + { + return compare; + } + + compare = CompareNullable(left.Kind, right.Kind); + if (compare != 0) + { + return compare; + } + + compare = CompareNullable(left.SourceTag, right.SourceTag); + if (compare != 0) + { + return compare; + } + + return left.Provenance.RecordedAt.CompareTo(right.Provenance.RecordedAt); + } + + private static int CompareNullable(string? left, string? right) + { + if (left is null && right is null) + { + return 0; + } + + if (left is null) + { + return 1; + } + + if (right is null) + { + return -1; + } + + return StringComparer.Ordinal.Compare(left, right); + } +} diff --git a/src/StellaOps.Feedser.Source.Osv/Internal/OsvVulnerabilityDto.cs b/src/StellaOps.Feedser.Source.Osv/Internal/OsvVulnerabilityDto.cs new file mode 100644 index 00000000..26f4ce12 --- /dev/null +++ b/src/StellaOps.Feedser.Source.Osv/Internal/OsvVulnerabilityDto.cs @@ -0,0 +1,129 @@ +using System; +using System.Collections.Generic; +using System.Text.Json; +using System.Text.Json.Serialization; + +namespace StellaOps.Feedser.Source.Osv.Internal; + +internal sealed record OsvVulnerabilityDto +{ + [JsonPropertyName("id")] + public string Id { get; init; } = string.Empty; + + [JsonPropertyName("summary")] + public string? Summary { get; init; } + + [JsonPropertyName("details")] + public string? Details { get; init; } + + [JsonPropertyName("aliases")] + public IReadOnlyList? Aliases { get; init; } + + [JsonPropertyName("related")] + public IReadOnlyList? Related { get; init; } + + [JsonPropertyName("published")] + public DateTimeOffset? Published { get; init; } + + [JsonPropertyName("modified")] + public DateTimeOffset? Modified { get; init; } + + [JsonPropertyName("severity")] + public IReadOnlyList? Severity { get; init; } + + [JsonPropertyName("references")] + public IReadOnlyList? References { get; init; } + + [JsonPropertyName("affected")] + public IReadOnlyList? Affected { get; init; } + + [JsonPropertyName("credits")] + public IReadOnlyList? Credits { get; init; } + + [JsonPropertyName("database_specific")] + public JsonElement DatabaseSpecific { get; init; } +} + +internal sealed record OsvSeverityDto +{ + [JsonPropertyName("type")] + public string? Type { get; init; } + + [JsonPropertyName("score")] + public string? Score { get; init; } +} + +internal sealed record OsvReferenceDto +{ + [JsonPropertyName("type")] + public string? Type { get; init; } + + [JsonPropertyName("url")] + public string? Url { get; init; } +} + +internal sealed record OsvCreditDto +{ + [JsonPropertyName("name")] + public string? Name { get; init; } + + [JsonPropertyName("type")] + public string? Type { get; init; } + + [JsonPropertyName("contact")] + public IReadOnlyList? Contact { get; init; } +} + +internal sealed record OsvAffectedPackageDto +{ + [JsonPropertyName("package")] + public OsvPackageDto? Package { get; init; } + + [JsonPropertyName("ranges")] + public IReadOnlyList? Ranges { get; init; } + + [JsonPropertyName("versions")] + public IReadOnlyList? Versions { get; init; } + + [JsonPropertyName("ecosystem_specific")] + public JsonElement EcosystemSpecific { get; init; } +} + +internal sealed record OsvPackageDto +{ + [JsonPropertyName("ecosystem")] + public string? Ecosystem { get; init; } + + [JsonPropertyName("name")] + public string? Name { get; init; } + + [JsonPropertyName("purl")] + public string? Purl { get; init; } +} + +internal sealed record OsvRangeDto +{ + [JsonPropertyName("type")] + public string? Type { get; init; } + + [JsonPropertyName("events")] + public IReadOnlyList? Events { get; init; } + + [JsonPropertyName("repo")] + public string? Repository { get; init; } +} + +internal sealed record OsvEventDto +{ + [JsonPropertyName("introduced")] + public string? Introduced { get; init; } + + [JsonPropertyName("fixed")] + public string? Fixed { get; init; } + + [JsonPropertyName("last_affected")] + public string? LastAffected { get; init; } + + [JsonPropertyName("limit")] + public string? Limit { get; init; } +} diff --git a/src/StellaOps.Feedser.Source.Osv/Jobs.cs b/src/StellaOps.Feedser.Source.Osv/Jobs.cs new file mode 100644 index 00000000..14e395d4 --- /dev/null +++ b/src/StellaOps.Feedser.Source.Osv/Jobs.cs @@ -0,0 +1,46 @@ +using System; +using System.Threading; +using System.Threading.Tasks; +using StellaOps.Feedser.Core.Jobs; + +namespace StellaOps.Feedser.Source.Osv; + +internal static class OsvJobKinds +{ + public const string Fetch = "source:osv:fetch"; + public const string Parse = "source:osv:parse"; + public const string Map = "source:osv:map"; +} + +internal sealed class OsvFetchJob : IJob +{ + private readonly OsvConnector _connector; + + public OsvFetchJob(OsvConnector connector) + => _connector = connector ?? throw new ArgumentNullException(nameof(connector)); + + public Task ExecuteAsync(JobExecutionContext context, CancellationToken cancellationToken) + => _connector.FetchAsync(context.Services, cancellationToken); +} + +internal sealed class OsvParseJob : IJob +{ + private readonly OsvConnector _connector; + + public OsvParseJob(OsvConnector connector) + => _connector = connector ?? throw new ArgumentNullException(nameof(connector)); + + public Task ExecuteAsync(JobExecutionContext context, CancellationToken cancellationToken) + => _connector.ParseAsync(context.Services, cancellationToken); +} + +internal sealed class OsvMapJob : IJob +{ + private readonly OsvConnector _connector; + + public OsvMapJob(OsvConnector connector) + => _connector = connector ?? throw new ArgumentNullException(nameof(connector)); + + public Task ExecuteAsync(JobExecutionContext context, CancellationToken cancellationToken) + => _connector.MapAsync(context.Services, cancellationToken); +} diff --git a/src/StellaOps.Feedser.Source.Osv/OsvConnector.cs b/src/StellaOps.Feedser.Source.Osv/OsvConnector.cs new file mode 100644 index 00000000..d15e3e54 --- /dev/null +++ b/src/StellaOps.Feedser.Source.Osv/OsvConnector.cs @@ -0,0 +1,500 @@ +using System; +using System.Collections.Generic; +using System.IO; +using System.IO.Compression; +using System.Linq; +using System.Net; +using System.Net.Http; +using System.Security.Cryptography; +using System.Text.Json; +using System.Text.Json.Serialization; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Options; +using MongoDB.Bson; +using MongoDB.Bson.IO; +using StellaOps.Feedser.Models; +using StellaOps.Feedser.Models; +using StellaOps.Feedser.Source.Common; +using StellaOps.Feedser.Source.Common.Fetch; +using StellaOps.Feedser.Source.Osv.Configuration; +using StellaOps.Feedser.Source.Osv.Internal; +using StellaOps.Feedser.Storage.Mongo; +using StellaOps.Feedser.Storage.Mongo.Advisories; +using StellaOps.Feedser.Storage.Mongo.Documents; +using StellaOps.Feedser.Storage.Mongo.Dtos; +using StellaOps.Plugin; + +namespace StellaOps.Feedser.Source.Osv; + +public sealed class OsvConnector : IFeedConnector +{ + private static readonly JsonSerializerOptions SerializerOptions = new(JsonSerializerDefaults.Web) + { + DefaultIgnoreCondition = JsonIgnoreCondition.WhenWritingNull, + PropertyNameCaseInsensitive = true, + }; + + private readonly IHttpClientFactory _httpClientFactory; + private readonly RawDocumentStorage _rawDocumentStorage; + private readonly IDocumentStore _documentStore; + private readonly IDtoStore _dtoStore; + private readonly IAdvisoryStore _advisoryStore; + private readonly ISourceStateRepository _stateRepository; + private readonly OsvOptions _options; + private readonly TimeProvider _timeProvider; + private readonly ILogger _logger; + + public OsvConnector( + IHttpClientFactory httpClientFactory, + RawDocumentStorage rawDocumentStorage, + IDocumentStore documentStore, + IDtoStore dtoStore, + IAdvisoryStore advisoryStore, + ISourceStateRepository stateRepository, + IOptions options, + TimeProvider? timeProvider, + ILogger logger) + { + _httpClientFactory = httpClientFactory ?? throw new ArgumentNullException(nameof(httpClientFactory)); + _rawDocumentStorage = rawDocumentStorage ?? throw new ArgumentNullException(nameof(rawDocumentStorage)); + _documentStore = documentStore ?? throw new ArgumentNullException(nameof(documentStore)); + _dtoStore = dtoStore ?? throw new ArgumentNullException(nameof(dtoStore)); + _advisoryStore = advisoryStore ?? throw new ArgumentNullException(nameof(advisoryStore)); + _stateRepository = stateRepository ?? throw new ArgumentNullException(nameof(stateRepository)); + _options = (options ?? throw new ArgumentNullException(nameof(options))).Value ?? throw new ArgumentNullException(nameof(options)); + _options.Validate(); + _timeProvider = timeProvider ?? TimeProvider.System; + _logger = logger ?? throw new ArgumentNullException(nameof(logger)); + } + + public string SourceName => OsvConnectorPlugin.SourceName; + + public async Task FetchAsync(IServiceProvider services, CancellationToken cancellationToken) + { + ArgumentNullException.ThrowIfNull(services); + + var cursor = await GetCursorAsync(cancellationToken).ConfigureAwait(false); + var now = _timeProvider.GetUtcNow(); + var pendingDocuments = cursor.PendingDocuments.ToHashSet(); + var cursorState = cursor; + var remainingCapacity = _options.MaxAdvisoriesPerFetch; + + foreach (var ecosystem in _options.Ecosystems) + { + if (remainingCapacity <= 0) + { + break; + } + + cancellationToken.ThrowIfCancellationRequested(); + + try + { + var result = await FetchEcosystemAsync( + ecosystem, + cursorState, + pendingDocuments, + now, + remainingCapacity, + cancellationToken).ConfigureAwait(false); + + cursorState = result.Cursor; + remainingCapacity -= result.NewDocuments; + } + catch (Exception ex) + { + _logger.LogError(ex, "OSV fetch failed for ecosystem {Ecosystem}", ecosystem); + await _stateRepository.MarkFailureAsync(SourceName, now, TimeSpan.FromMinutes(10), ex.Message, cancellationToken).ConfigureAwait(false); + throw; + } + } + + cursorState = cursorState + .WithPendingDocuments(pendingDocuments) + .WithPendingMappings(cursor.PendingMappings); + + await UpdateCursorAsync(cursorState, cancellationToken).ConfigureAwait(false); + } + + public async Task ParseAsync(IServiceProvider services, CancellationToken cancellationToken) + { + ArgumentNullException.ThrowIfNull(services); + + var cursor = await GetCursorAsync(cancellationToken).ConfigureAwait(false); + if (cursor.PendingDocuments.Count == 0) + { + return; + } + + var remainingDocuments = cursor.PendingDocuments.ToList(); + var pendingMappings = cursor.PendingMappings.ToList(); + + foreach (var documentId in cursor.PendingDocuments) + { + cancellationToken.ThrowIfCancellationRequested(); + + var document = await _documentStore.FindAsync(documentId, cancellationToken).ConfigureAwait(false); + if (document is null) + { + remainingDocuments.Remove(documentId); + continue; + } + + if (!document.GridFsId.HasValue) + { + _logger.LogWarning("OSV document {DocumentId} missing GridFS content", document.Id); + await _documentStore.UpdateStatusAsync(document.Id, DocumentStatuses.Failed, cancellationToken).ConfigureAwait(false); + remainingDocuments.Remove(documentId); + continue; + } + + byte[] bytes; + try + { + bytes = await _rawDocumentStorage.DownloadAsync(document.GridFsId.Value, cancellationToken).ConfigureAwait(false); + } + catch (Exception ex) + { + _logger.LogError(ex, "Unable to download OSV raw document {DocumentId}", document.Id); + throw; + } + + OsvVulnerabilityDto? dto; + try + { + dto = JsonSerializer.Deserialize(bytes, SerializerOptions); + } + catch (Exception ex) + { + _logger.LogWarning(ex, "Failed to deserialize OSV document {DocumentId} ({Uri})", document.Id, document.Uri); + await _documentStore.UpdateStatusAsync(document.Id, DocumentStatuses.Failed, cancellationToken).ConfigureAwait(false); + remainingDocuments.Remove(documentId); + continue; + } + + if (dto is null || string.IsNullOrWhiteSpace(dto.Id)) + { + _logger.LogWarning("OSV document {DocumentId} produced empty payload", document.Id); + await _documentStore.UpdateStatusAsync(document.Id, DocumentStatuses.Failed, cancellationToken).ConfigureAwait(false); + remainingDocuments.Remove(documentId); + continue; + } + + var sanitized = JsonSerializer.Serialize(dto, SerializerOptions); + var payload = MongoDB.Bson.BsonDocument.Parse(sanitized); + var dtoRecord = new DtoRecord( + Guid.NewGuid(), + document.Id, + SourceName, + "osv.v1", + payload, + _timeProvider.GetUtcNow()); + + await _dtoStore.UpsertAsync(dtoRecord, cancellationToken).ConfigureAwait(false); + await _documentStore.UpdateStatusAsync(document.Id, DocumentStatuses.PendingMap, cancellationToken).ConfigureAwait(false); + + remainingDocuments.Remove(documentId); + if (!pendingMappings.Contains(documentId)) + { + pendingMappings.Add(documentId); + } + } + + var updatedCursor = cursor + .WithPendingDocuments(remainingDocuments) + .WithPendingMappings(pendingMappings); + + await UpdateCursorAsync(updatedCursor, cancellationToken).ConfigureAwait(false); + } + + public async Task MapAsync(IServiceProvider services, CancellationToken cancellationToken) + { + ArgumentNullException.ThrowIfNull(services); + + var cursor = await GetCursorAsync(cancellationToken).ConfigureAwait(false); + if (cursor.PendingMappings.Count == 0) + { + return; + } + + var pendingMappings = cursor.PendingMappings.ToList(); + + foreach (var documentId in cursor.PendingMappings) + { + cancellationToken.ThrowIfCancellationRequested(); + + var dto = await _dtoStore.FindByDocumentIdAsync(documentId, cancellationToken).ConfigureAwait(false); + var document = await _documentStore.FindAsync(documentId, cancellationToken).ConfigureAwait(false); + + if (dto is null || document is null) + { + pendingMappings.Remove(documentId); + continue; + } + + var payloadJson = dto.Payload.ToJson(new JsonWriterSettings + { + OutputMode = JsonOutputMode.RelaxedExtendedJson, + }); + + OsvVulnerabilityDto? osvDto; + try + { + osvDto = JsonSerializer.Deserialize(payloadJson, SerializerOptions); + } + catch (Exception ex) + { + _logger.LogError(ex, "Failed to deserialize OSV DTO for document {DocumentId}", document.Id); + await _documentStore.UpdateStatusAsync(document.Id, DocumentStatuses.Failed, cancellationToken).ConfigureAwait(false); + pendingMappings.Remove(documentId); + continue; + } + + if (osvDto is null || string.IsNullOrWhiteSpace(osvDto.Id)) + { + await _documentStore.UpdateStatusAsync(document.Id, DocumentStatuses.Failed, cancellationToken).ConfigureAwait(false); + pendingMappings.Remove(documentId); + continue; + } + + var ecosystem = document.Metadata is not null && document.Metadata.TryGetValue("osv.ecosystem", out var ecosystemValue) + ? ecosystemValue + : "unknown"; + + var advisory = OsvMapper.Map(osvDto, document, dto, ecosystem); + await _advisoryStore.UpsertAsync(advisory, cancellationToken).ConfigureAwait(false); + await _documentStore.UpdateStatusAsync(document.Id, DocumentStatuses.Mapped, cancellationToken).ConfigureAwait(false); + + pendingMappings.Remove(documentId); + } + + var updatedCursor = cursor.WithPendingMappings(pendingMappings); + await UpdateCursorAsync(updatedCursor, cancellationToken).ConfigureAwait(false); + } + + private async Task GetCursorAsync(CancellationToken cancellationToken) + { + var state = await _stateRepository.TryGetAsync(SourceName, cancellationToken).ConfigureAwait(false); + return state is null ? OsvCursor.Empty : OsvCursor.FromBson(state.Cursor); + } + + private async Task UpdateCursorAsync(OsvCursor cursor, CancellationToken cancellationToken) + { + var document = cursor.ToBsonDocument(); + await _stateRepository.UpdateCursorAsync(SourceName, document, _timeProvider.GetUtcNow(), cancellationToken).ConfigureAwait(false); + } + + private async Task<(OsvCursor Cursor, int NewDocuments)> FetchEcosystemAsync( + string ecosystem, + OsvCursor cursor, + HashSet pendingDocuments, + DateTimeOffset now, + int remainingCapacity, + CancellationToken cancellationToken) + { + var client = _httpClientFactory.CreateClient(OsvOptions.HttpClientName); + client.Timeout = _options.HttpTimeout; + + var archiveUri = BuildArchiveUri(ecosystem); + using var request = new HttpRequestMessage(HttpMethod.Get, archiveUri); + + if (cursor.TryGetArchiveMetadata(ecosystem, out var archiveMetadata)) + { + if (!string.IsNullOrWhiteSpace(archiveMetadata.ETag)) + { + request.Headers.TryAddWithoutValidation("If-None-Match", archiveMetadata.ETag); + } + + if (archiveMetadata.LastModified.HasValue) + { + request.Headers.IfModifiedSince = archiveMetadata.LastModified.Value; + } + } + + using var response = await client.SendAsync(request, HttpCompletionOption.ResponseHeadersRead, cancellationToken).ConfigureAwait(false); + + if (response.StatusCode == HttpStatusCode.NotModified) + { + return (cursor, 0); + } + + response.EnsureSuccessStatusCode(); + + await using var archiveStream = await response.Content.ReadAsStreamAsync(cancellationToken).ConfigureAwait(false); + using var archive = new ZipArchive(archiveStream, ZipArchiveMode.Read, leaveOpen: false); + + var existingLastModified = cursor.GetLastModified(ecosystem); + var processedIdsSet = cursor.ProcessedIdsByEcosystem.TryGetValue(ecosystem, out var processedIds) + ? new HashSet(processedIds, StringComparer.OrdinalIgnoreCase) + : new HashSet(StringComparer.OrdinalIgnoreCase); + + var currentMaxModified = existingLastModified ?? DateTimeOffset.MinValue; + var currentProcessedIds = new HashSet(processedIdsSet, StringComparer.OrdinalIgnoreCase); + var processedUpdated = false; + var newDocuments = 0; + + var minimumModified = existingLastModified.HasValue + ? existingLastModified.Value - _options.ModifiedTolerance + : now - _options.InitialBackfill; + + ProvenanceDiagnostics.ReportResumeWindow(SourceName, minimumModified, _logger); + + foreach (var entry in archive.Entries) + { + if (remainingCapacity <= 0) + { + break; + } + + cancellationToken.ThrowIfCancellationRequested(); + + if (!entry.FullName.EndsWith(".json", StringComparison.OrdinalIgnoreCase)) + { + continue; + } + + await using var entryStream = entry.Open(); + using var memory = new MemoryStream(); + await entryStream.CopyToAsync(memory, cancellationToken).ConfigureAwait(false); + var bytes = memory.ToArray(); + + OsvVulnerabilityDto? dto; + try + { + dto = JsonSerializer.Deserialize(bytes, SerializerOptions); + } + catch (Exception ex) + { + _logger.LogWarning(ex, "Failed to parse OSV entry {Entry} for ecosystem {Ecosystem}", entry.FullName, ecosystem); + continue; + } + + if (dto is null || string.IsNullOrWhiteSpace(dto.Id)) + { + continue; + } + + var modified = (dto.Modified ?? dto.Published ?? DateTimeOffset.MinValue).ToUniversalTime(); + if (modified < minimumModified) + { + continue; + } + + if (existingLastModified.HasValue && modified < existingLastModified.Value - _options.ModifiedTolerance) + { + continue; + } + + if (modified < currentMaxModified - _options.ModifiedTolerance) + { + continue; + } + + if (modified == currentMaxModified && currentProcessedIds.Contains(dto.Id)) + { + continue; + } + + var documentUri = BuildDocumentUri(ecosystem, dto.Id); + var sha256 = Convert.ToHexString(SHA256.HashData(bytes)).ToLowerInvariant(); + + var existing = await _documentStore.FindBySourceAndUriAsync(SourceName, documentUri, cancellationToken).ConfigureAwait(false); + if (existing is not null && string.Equals(existing.Sha256, sha256, StringComparison.OrdinalIgnoreCase)) + { + continue; + } + + var gridFsId = await _rawDocumentStorage.UploadAsync(SourceName, documentUri, bytes, "application/json", null, cancellationToken).ConfigureAwait(false); + var metadata = new Dictionary(StringComparer.Ordinal) + { + ["osv.ecosystem"] = ecosystem, + ["osv.id"] = dto.Id, + ["osv.modified"] = modified.ToString("O"), + }; + + var recordId = existing?.Id ?? Guid.NewGuid(); + var record = new DocumentRecord( + recordId, + SourceName, + documentUri, + _timeProvider.GetUtcNow(), + sha256, + DocumentStatuses.PendingParse, + "application/json", + Headers: null, + Metadata: metadata, + Etag: null, + LastModified: modified, + GridFsId: gridFsId, + ExpiresAt: null); + + var upserted = await _documentStore.UpsertAsync(record, cancellationToken).ConfigureAwait(false); + pendingDocuments.Add(upserted.Id); + newDocuments++; + remainingCapacity--; + + if (modified > currentMaxModified) + { + currentMaxModified = modified; + currentProcessedIds = new HashSet(StringComparer.OrdinalIgnoreCase) { dto.Id }; + processedUpdated = true; + } + else if (modified == currentMaxModified) + { + currentProcessedIds.Add(dto.Id); + processedUpdated = true; + } + + if (_options.RequestDelay > TimeSpan.Zero) + { + try + { + await Task.Delay(_options.RequestDelay, cancellationToken).ConfigureAwait(false); + } + catch (TaskCanceledException) + { + break; + } + } + } + + if (processedUpdated && currentMaxModified != DateTimeOffset.MinValue) + { + cursor = cursor.WithLastModified(ecosystem, currentMaxModified, currentProcessedIds); + } + else if (processedUpdated && existingLastModified.HasValue) + { + cursor = cursor.WithLastModified(ecosystem, existingLastModified.Value, currentProcessedIds); + } + + var etag = response.Headers.ETag?.Tag; + var lastModifiedHeader = response.Content.Headers.LastModified; + cursor = cursor.WithArchiveMetadata(ecosystem, etag, lastModifiedHeader); + + return (cursor, newDocuments); + } + + private Uri BuildArchiveUri(string ecosystem) + { + var trimmed = ecosystem.Trim('/'); + var baseUri = _options.BaseUri; + var builder = new UriBuilder(baseUri); + var path = builder.Path; + if (!path.EndsWith('/')) + { + path += "/"; + } + + path += $"{trimmed}/{_options.ArchiveFileName}"; + builder.Path = path; + return builder.Uri; + } + + private static string BuildDocumentUri(string ecosystem, string vulnerabilityId) + { + var safeId = vulnerabilityId.Replace(' ', '-'); + return $"https://osv-vulnerabilities.storage.googleapis.com/{ecosystem}/{safeId}.json"; + } +} diff --git a/src/StellaOps.Feedser.Source.Osv/OsvConnectorPlugin.cs b/src/StellaOps.Feedser.Source.Osv/OsvConnectorPlugin.cs new file mode 100644 index 00000000..a5beba0f --- /dev/null +++ b/src/StellaOps.Feedser.Source.Osv/OsvConnectorPlugin.cs @@ -0,0 +1,20 @@ +using System; +using Microsoft.Extensions.DependencyInjection; +using StellaOps.Plugin; + +namespace StellaOps.Feedser.Source.Osv; + +public sealed class OsvConnectorPlugin : IConnectorPlugin +{ + public string Name => SourceName; + + public static string SourceName => "osv"; + + public bool IsAvailable(IServiceProvider services) => services is not null; + + public IFeedConnector Create(IServiceProvider services) + { + ArgumentNullException.ThrowIfNull(services); + return ActivatorUtilities.CreateInstance(services); + } +} diff --git a/src/StellaOps.Feedser.Source.Osv/OsvDependencyInjectionRoutine.cs b/src/StellaOps.Feedser.Source.Osv/OsvDependencyInjectionRoutine.cs new file mode 100644 index 00000000..c401aaa2 --- /dev/null +++ b/src/StellaOps.Feedser.Source.Osv/OsvDependencyInjectionRoutine.cs @@ -0,0 +1,53 @@ +using System; +using Microsoft.Extensions.Configuration; +using Microsoft.Extensions.DependencyInjection; +using StellaOps.DependencyInjection; +using StellaOps.Feedser.Core.Jobs; +using StellaOps.Feedser.Source.Osv.Configuration; + +namespace StellaOps.Feedser.Source.Osv; + +public sealed class OsvDependencyInjectionRoutine : IDependencyInjectionRoutine +{ + private const string ConfigurationSection = "feedser:sources:osv"; + private const string FetchCron = "0,20,40 * * * *"; + private const string ParseCron = "5,25,45 * * * *"; + private const string MapCron = "10,30,50 * * * *"; + + private static readonly TimeSpan FetchTimeout = TimeSpan.FromMinutes(15); + private static readonly TimeSpan ParseTimeout = TimeSpan.FromMinutes(20); + private static readonly TimeSpan MapTimeout = TimeSpan.FromMinutes(20); + private static readonly TimeSpan LeaseDuration = TimeSpan.FromMinutes(10); + + public IServiceCollection Register(IServiceCollection services, IConfiguration configuration) + { + ArgumentNullException.ThrowIfNull(services); + ArgumentNullException.ThrowIfNull(configuration); + + services.AddOsvConnector(options => + { + configuration.GetSection(ConfigurationSection).Bind(options); + options.Validate(); + }); + + var scheduler = new JobSchedulerBuilder(services); + scheduler + .AddJob( + OsvJobKinds.Fetch, + cronExpression: FetchCron, + timeout: FetchTimeout, + leaseDuration: LeaseDuration) + .AddJob( + OsvJobKinds.Parse, + cronExpression: ParseCron, + timeout: ParseTimeout, + leaseDuration: LeaseDuration) + .AddJob( + OsvJobKinds.Map, + cronExpression: MapCron, + timeout: MapTimeout, + leaseDuration: LeaseDuration); + + return services; + } +} diff --git a/src/StellaOps.Feedser.Source.Osv/OsvServiceCollectionExtensions.cs b/src/StellaOps.Feedser.Source.Osv/OsvServiceCollectionExtensions.cs new file mode 100644 index 00000000..269bdb5a --- /dev/null +++ b/src/StellaOps.Feedser.Source.Osv/OsvServiceCollectionExtensions.cs @@ -0,0 +1,37 @@ +using System; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Options; +using StellaOps.Feedser.Source.Common.Http; +using StellaOps.Feedser.Source.Osv.Configuration; + +namespace StellaOps.Feedser.Source.Osv; + +public static class OsvServiceCollectionExtensions +{ + public static IServiceCollection AddOsvConnector(this IServiceCollection services, Action configure) + { + ArgumentNullException.ThrowIfNull(services); + ArgumentNullException.ThrowIfNull(configure); + + services.AddOptions() + .Configure(configure) + .PostConfigure(static opts => opts.Validate()); + + services.AddSourceHttpClient(OsvOptions.HttpClientName, (sp, clientOptions) => + { + var options = sp.GetRequiredService>().Value; + clientOptions.BaseAddress = options.BaseUri; + clientOptions.Timeout = options.HttpTimeout; + clientOptions.UserAgent = "StellaOps.Feedser.OSV/1.0"; + clientOptions.AllowedHosts.Clear(); + clientOptions.AllowedHosts.Add(options.BaseUri.Host); + clientOptions.DefaultRequestHeaders["Accept"] = "application/zip"; + }); + + services.AddTransient(); + services.AddTransient(); + services.AddTransient(); + services.AddTransient(); + return services; + } +} diff --git a/src/StellaOps.Feedser.Source.Osv/Properties/AssemblyInfo.cs b/src/StellaOps.Feedser.Source.Osv/Properties/AssemblyInfo.cs new file mode 100644 index 00000000..150bab12 --- /dev/null +++ b/src/StellaOps.Feedser.Source.Osv/Properties/AssemblyInfo.cs @@ -0,0 +1,3 @@ +using System.Runtime.CompilerServices; + +[assembly: InternalsVisibleTo("FixtureUpdater")] diff --git a/src/StellaOps.Feedser.Source.Osv/StellaOps.Feedser.Source.Osv.csproj b/src/StellaOps.Feedser.Source.Osv/StellaOps.Feedser.Source.Osv.csproj new file mode 100644 index 00000000..a3256c0b --- /dev/null +++ b/src/StellaOps.Feedser.Source.Osv/StellaOps.Feedser.Source.Osv.csproj @@ -0,0 +1,23 @@ + + + net10.0 + enable + enable + + + + + + + + + + + + <_Parameter1>StellaOps.Feedser.Tests + + + <_Parameter1>StellaOps.Feedser.Source.Osv.Tests + + + diff --git a/src/StellaOps.Feedser.Source.Osv/TASKS.md b/src/StellaOps.Feedser.Source.Osv/TASKS.md new file mode 100644 index 00000000..4bda9aef --- /dev/null +++ b/src/StellaOps.Feedser.Source.Osv/TASKS.md @@ -0,0 +1,15 @@ +# TASKS +| Task | Owner(s) | Depends on | Notes | +|---|---|---|---| +|Ecosystem fetchers (npm, pypi, maven, go, crates)|BE-Conn-OSV|Source.Common|**DONE** – archive fetch loop iterates ecosystems with pagination + change gating.| +|OSV options & HttpClient configuration|BE-Conn-OSV|Source.Common|**DONE** – `OsvOptions` + `AddOsvConnector` configure allowlisted HttpClient.| +|DTO validation + sanitizer|BE-Conn-OSV|Source.Common|**DONE** – JSON deserialization sanitizes payloads before persistence; schema enforcement deferred.| +|Mapper to canonical SemVer ranges|BE-Conn-OSV|Models|**DONE** – `OsvMapper` emits SemVer ranges with provenance metadata.| +|Alias consolidation (GHSA/CVE)|BE-Merge|Merge|DONE – OSV advisory records now emit GHSA/CVE aliases captured by alias graph tests.| +|Tests: snapshot per ecosystem|QA|Tests|DONE – deterministic snapshots added for npm and PyPI advisories.| +|Cursor persistence and hash gating|BE-Conn-OSV|Storage.Mongo|**DONE** – `OsvCursor` tracks per-ecosystem metadata and SHA gating.| +|Parity checks vs GHSA data|QA|Merge|DONE – `OsvGhsaParityRegressionTests` keep OSV ↔ GHSA fixtures green; regeneration workflow documented in docs/19_TEST_SUITE_OVERVIEW.md.| +|Connector DI routine & job registration|BE-Conn-OSV|Core|**DONE** – DI routine registers fetch/parse/map jobs with scheduler.| +|Implement OSV fetch/parse/map skeleton|BE-Conn-OSV|Source.Common|**DONE** – connector now persists documents, DTOs, and canonical advisories.| +|FEEDCONN-OSV-02-004 OSV references & credits alignment|BE-Conn-OSV|Models `FEEDMODELS-SCHEMA-01-002`|**DONE (2025-10-11)** – Mapper normalizes references with provenance masks, emits advisory credits, and regression fixtures/assertions cover the new fields.| +|FEEDCONN-OSV-02-005 Fixture updater workflow|BE-Conn-OSV, QA|Docs|TODO – Document `tools/FixtureUpdater`, add parity regression steps, and ensure future refreshes capture credit metadata consistently.| diff --git a/src/StellaOps.Feedser.Source.Ru.Bdu/AGENTS.md b/src/StellaOps.Feedser.Source.Ru.Bdu/AGENTS.md new file mode 100644 index 00000000..17a9d357 --- /dev/null +++ b/src/StellaOps.Feedser.Source.Ru.Bdu/AGENTS.md @@ -0,0 +1,38 @@ +# AGENTS +## Role +Implement the Russian BDU (Vulnerability Database) connector to ingest advisories published by FSTEC’s BDU catalogue. + +## Scope +- Determine accessible BDU feeds/APIs (HTML listings, downloadable CSV, SOAP/REST) and access constraints. +- Build fetch/cursor pipeline with dedupe, retries, and backoff appropriate for the data source. +- Parse advisory records to extract summary, affected vendors/products, mitigation recommendations, CVE IDs. +- Map advisories into canonical `Advisory` objects including aliases, references, affected packages, and range primitives. +- Provide deterministic fixtures and regression tests for the connector lifecycle. + +## Participants +- `Source.Common` (HTTP/fetch utilities, DTO storage). +- `Storage.Mongo` (raw/document/DTO/advisory stores + source state). +- `Feedser.Models` (canonical data structures). +- `Feedser.Testing` (integration harness, snapshot utilities). + +## Interfaces & Contracts +- Job kinds: `bdu:fetch`, `bdu:parse`, `bdu:map`. +- Persist upstream metadata (e.g., record modification timestamp) to drive incremental updates. +- Alias set should include BDU identifiers and CVE IDs when present. + +## In/Out of scope +In scope: +- Core ingestion/mapping of BDU vulnerability records. + +Out of scope: +- Translation beyond normalising required canonical fields. + +## Observability & Security Expectations +- Log fetch/mapping statistics and failure details. +- Sanitize source payloads, handling Cyrillic text/encodings correctly. +- Respect upstream rate limits and mark failures with backoff. + +## Tests +- Add `StellaOps.Feedser.Source.Ru.Bdu.Tests` covering fetch/parse/map with canned fixtures. +- Snapshot canonical advisories; support fixture regeneration via env flag. +- Ensure deterministic ordering/time normalisation. diff --git a/src/StellaOps.Feedser.Source.Ru.Bdu/Class1.cs b/src/StellaOps.Feedser.Source.Ru.Bdu/Class1.cs new file mode 100644 index 00000000..6a891d46 --- /dev/null +++ b/src/StellaOps.Feedser.Source.Ru.Bdu/Class1.cs @@ -0,0 +1,29 @@ +using System; +using System.Threading; +using System.Threading.Tasks; +using StellaOps.Plugin; + +namespace StellaOps.Feedser.Source.Ru.Bdu; + +public sealed class RuBduConnectorPlugin : IConnectorPlugin +{ + public string Name => "ru-bdu"; + + public bool IsAvailable(IServiceProvider services) => true; + + public IFeedConnector Create(IServiceProvider services) => new StubConnector(Name); + + private sealed class StubConnector : IFeedConnector + { + public StubConnector(string sourceName) => SourceName = sourceName; + + public string SourceName { get; } + + public Task FetchAsync(IServiceProvider services, CancellationToken cancellationToken) => Task.CompletedTask; + + public Task ParseAsync(IServiceProvider services, CancellationToken cancellationToken) => Task.CompletedTask; + + public Task MapAsync(IServiceProvider services, CancellationToken cancellationToken) => Task.CompletedTask; + } +} + diff --git a/src/StellaOps.Feedser.Source.Ru.Bdu/StellaOps.Feedser.Source.Ru.Bdu.csproj b/src/StellaOps.Feedser.Source.Ru.Bdu/StellaOps.Feedser.Source.Ru.Bdu.csproj new file mode 100644 index 00000000..f7f2c154 --- /dev/null +++ b/src/StellaOps.Feedser.Source.Ru.Bdu/StellaOps.Feedser.Source.Ru.Bdu.csproj @@ -0,0 +1,16 @@ + + + + net10.0 + enable + enable + + + + + + + + + + diff --git a/src/StellaOps.Feedser.Source.Ru.Bdu/TASKS.md b/src/StellaOps.Feedser.Source.Ru.Bdu/TASKS.md new file mode 100644 index 00000000..fce896a6 --- /dev/null +++ b/src/StellaOps.Feedser.Source.Ru.Bdu/TASKS.md @@ -0,0 +1,11 @@ +# TASKS +| Task | Owner(s) | Depends on | Notes | +|---|---|---|---| +|FEEDCONN-RUBDU-02-001 Identify BDU data source & schema|BE-Conn-BDU|Research|**DOING** – 2025-10-11: BDU portal documents RSS/Atom feeds for vulnerabilities/threats; direct HTTPS pulls fail CA validation (self-signed chain) so we need to confirm auth, required trust anchors, and capture schema samples for `ubi/vul` endpoints.| +|FEEDCONN-RUBDU-02-002 Fetch pipeline & cursor handling|BE-Conn-BDU|Source.Common, Storage.Mongo|**TODO** – Implement fetch job with retry/backoff, persist raw documents and update source state.| +|FEEDCONN-RUBDU-02-003 DTO/parser implementation|BE-Conn-BDU|Source.Common|**TODO** – Create DTOs for BDU records (title, severity, vendor/product, references, CVEs); sanitise text.| +|FEEDCONN-RUBDU-02-004 Canonical mapping & range primitives|BE-Conn-BDU|Models|**TODO** – Map into canonical advisories with aliases, references, and vendor range primitives. Use normalized rule checkpoints from `../StellaOps.Feedser.Merge/RANGE_PRIMITIVES_COORDINATION.md`.| +|FEEDCONN-RUBDU-02-005 Deterministic fixtures & regression tests|QA|Testing|**TODO** – Add fetch/parse/map tests with fixtures; support `UPDATE_BDU_FIXTURES=1`.| +|FEEDCONN-RUBDU-02-006 Telemetry & documentation|DevEx|Docs|**TODO** – Add logging/metrics, document connector configuration, close backlog when complete.| +|FEEDCONN-RUBDU-02-007 Access & export options assessment|BE-Conn-BDU|Research|**TODO** – Evaluate HTML vs CSV downloads, session requirements, and plan cursor strategy for incremental pulls from bdu.fstec.ru.| +|FEEDCONN-RUBDU-02-008 Trusted root onboarding plan|BE-Conn-BDU|Source.Common|**BLOCKED** – 2025-10-11: Attempt to download Russian Trusted Sub CA returned placeholder HTML; need alternate distribution (mirror or manual bundle) before TLS validation succeeds.| diff --git a/src/StellaOps.Feedser.Source.Ru.Nkcki/AGENTS.md b/src/StellaOps.Feedser.Source.Ru.Nkcki/AGENTS.md new file mode 100644 index 00000000..00b65152 --- /dev/null +++ b/src/StellaOps.Feedser.Source.Ru.Nkcki/AGENTS.md @@ -0,0 +1,38 @@ +# AGENTS +## Role +Implement the Russian NKTsKI (formerly NKCKI) advisories connector to ingest NKTsKI vulnerability bulletins for Feedser’s regional coverage. + +## Scope +- Identify NKTsKI advisory feeds/APIs (HTML, RSS, CSV) and access/authentication requirements. +- Implement fetch/cursor pipeline with dedupe and failure backoff tailored to the source format. +- Parse advisories to extract summary, affected vendors/products, recommended mitigation, and CVE identifiers. +- Map advisories into canonical `Advisory` records with aliases, references, affected packages, and range primitives. +- Create deterministic fixtures and regression tests. + +## Participants +- `Source.Common` (HTTP/fetch utilities, DTO storage). +- `Storage.Mongo` (raw/document/DTO/advisory stores, source state). +- `Feedser.Models` (canonical data structures). +- `Feedser.Testing` (integration fixtures, snapshots). + +## Interfaces & Contracts +- Job kinds: `nkcki:fetch`, `nkcki:parse`, `nkcki:map`. +- Persist upstream modification metadata to support incremental updates. +- Alias set should include NKTsKI advisory IDs and CVEs when present. + +## In/Out of scope +In scope: +- Core ingestion/mapping pipeline with range primitives. + +Out of scope: +- Translation beyond canonical field normalisation. + +## Observability & Security Expectations +- Log fetch/mapping activity; mark failures with backoff delays. +- Handle Cyrillic text encoding and sanitise HTML safely. +- Respect upstream rate limiting/politeness. + +## Tests +- Add `StellaOps.Feedser.Source.Ru.Nkcki.Tests` for fetch/parse/map with canned fixtures. +- Snapshot canonical advisories; support fixture regeneration via env flag. +- Ensure deterministic ordering/time normalisation. diff --git a/src/StellaOps.Feedser.Source.Ru.Nkcki/Class1.cs b/src/StellaOps.Feedser.Source.Ru.Nkcki/Class1.cs new file mode 100644 index 00000000..c3f57c28 --- /dev/null +++ b/src/StellaOps.Feedser.Source.Ru.Nkcki/Class1.cs @@ -0,0 +1,29 @@ +using System; +using System.Threading; +using System.Threading.Tasks; +using StellaOps.Plugin; + +namespace StellaOps.Feedser.Source.Ru.Nkcki; + +public sealed class RuNkckiConnectorPlugin : IConnectorPlugin +{ + public string Name => "ru-nkcki"; + + public bool IsAvailable(IServiceProvider services) => true; + + public IFeedConnector Create(IServiceProvider services) => new StubConnector(Name); + + private sealed class StubConnector : IFeedConnector + { + public StubConnector(string sourceName) => SourceName = sourceName; + + public string SourceName { get; } + + public Task FetchAsync(IServiceProvider services, CancellationToken cancellationToken) => Task.CompletedTask; + + public Task ParseAsync(IServiceProvider services, CancellationToken cancellationToken) => Task.CompletedTask; + + public Task MapAsync(IServiceProvider services, CancellationToken cancellationToken) => Task.CompletedTask; + } +} + diff --git a/src/StellaOps.Feedser.Source.Ru.Nkcki/StellaOps.Feedser.Source.Ru.Nkcki.csproj b/src/StellaOps.Feedser.Source.Ru.Nkcki/StellaOps.Feedser.Source.Ru.Nkcki.csproj new file mode 100644 index 00000000..f7f2c154 --- /dev/null +++ b/src/StellaOps.Feedser.Source.Ru.Nkcki/StellaOps.Feedser.Source.Ru.Nkcki.csproj @@ -0,0 +1,16 @@ + + + + net10.0 + enable + enable + + + + + + + + + + diff --git a/src/StellaOps.Feedser.Source.Ru.Nkcki/TASKS.md b/src/StellaOps.Feedser.Source.Ru.Nkcki/TASKS.md new file mode 100644 index 00000000..ba4656b9 --- /dev/null +++ b/src/StellaOps.Feedser.Source.Ru.Nkcki/TASKS.md @@ -0,0 +1,11 @@ +# TASKS +| Task | Owner(s) | Depends on | Notes | +|---|---|---|---| +|FEEDCONN-NKCKI-02-001 Research NKTsKI advisory feeds|BE-Conn-Nkcki|Research|**DOING** – 2025-10-11: Located cert.gov.ru RSS landing page with paginated advisories; raw feed pulls currently blocked by TLS/403 so we still need to determine feed endpoints, auth requirements, and archive depth.| +|FEEDCONN-NKCKI-02-002 Fetch pipeline & state persistence|BE-Conn-Nkcki|Source.Common, Storage.Mongo|**TODO** – Implement fetch job with retry/backoff, persist raw documents, update cursor metadata.| +|FEEDCONN-NKCKI-02-003 DTO & parser implementation|BE-Conn-Nkcki|Source.Common|**TODO** – Build DTOs for NKTsKI advisories, sanitise HTML, extract vendors/products, CVEs, mitigation guidance.| +|FEEDCONN-NKCKI-02-004 Canonical mapping & range primitives|BE-Conn-Nkcki|Models|**TODO** – Map advisories into canonical records with aliases, references, and vendor range primitives. Coordinate normalized outputs and provenance per `../StellaOps.Feedser.Merge/RANGE_PRIMITIVES_COORDINATION.md`.| +|FEEDCONN-NKCKI-02-005 Deterministic fixtures & tests|QA|Testing|**TODO** – Add regression tests supporting `UPDATE_NKCKI_FIXTURES=1` for snapshot regeneration.| +|FEEDCONN-NKCKI-02-006 Telemetry & documentation|DevEx|Docs|**TODO** – Add logging/metrics, document connector configuration, and close backlog entry after deliverable ships.| +|FEEDCONN-NKCKI-02-007 Archive ingestion strategy|BE-Conn-Nkcki|Research|**TODO** – Document bulletin taxonomy (alerts vs recommendations), paging behaviour, and fallback scraping plan for legacy advisories.| +|FEEDCONN-NKCKI-02-008 Access enablement plan|BE-Conn-Nkcki|Source.Common|**TODO** – Determine certificate/proxy requirements to fetch cert.gov.ru RSS feeds without 403/TLS failures and outline monitoring for future access policy shifts.| diff --git a/src/StellaOps.Feedser.Source.Vndr.Adobe.Tests/Adobe/AdobeConnectorFetchTests.cs b/src/StellaOps.Feedser.Source.Vndr.Adobe.Tests/Adobe/AdobeConnectorFetchTests.cs new file mode 100644 index 00000000..47a10956 --- /dev/null +++ b/src/StellaOps.Feedser.Source.Vndr.Adobe.Tests/Adobe/AdobeConnectorFetchTests.cs @@ -0,0 +1,440 @@ +using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using System.Net; +using System.Net.Http; +using System.Net.Http.Headers; +using System.Text; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Http; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Logging.Abstractions; +using Microsoft.Extensions.Options; +using Microsoft.Extensions.Time.Testing; +using MongoDB.Bson; +using MongoDB.Driver; +using StellaOps.Feedser.Models; +using StellaOps.Feedser.Source.Common; +using StellaOps.Feedser.Source.Common.Http; +using StellaOps.Feedser.Source.Common.Testing; +using StellaOps.Feedser.Source.Vndr.Adobe; +using StellaOps.Feedser.Source.Vndr.Adobe.Configuration; +using StellaOps.Feedser.Storage.Mongo; +using StellaOps.Feedser.Storage.Mongo.Advisories; +using StellaOps.Feedser.Storage.Mongo.Documents; +using StellaOps.Feedser.Storage.Mongo.Dtos; +using StellaOps.Feedser.Storage.Mongo.PsirtFlags; +using StellaOps.Feedser.Testing; + +namespace StellaOps.Feedser.Source.Vndr.Adobe.Tests; + +[Collection("mongo-fixture")] +public sealed class AdobeConnectorFetchTests : IAsyncLifetime +{ + private readonly MongoIntegrationFixture _fixture; + private readonly FakeTimeProvider _timeProvider; + + public AdobeConnectorFetchTests(MongoIntegrationFixture fixture) + { + _fixture = fixture; + _timeProvider = new FakeTimeProvider(new DateTimeOffset(2025, 9, 10, 0, 0, 0, TimeSpan.Zero)); + } + + [Fact] + public async Task Fetch_WindowsIndexAndPersistsCursor() + { + var handler = new CannedHttpMessageHandler(); + await using var provider = await BuildServiceProviderAsync(handler); + SeedIndex(handler); + SeedDetail(handler); + + var connector = provider.GetRequiredService(); + await connector.FetchAsync(provider, CancellationToken.None); + + var stateRepository = provider.GetRequiredService(); + var state = await stateRepository.TryGetAsync(VndrAdobeConnectorPlugin.SourceName, CancellationToken.None); + Assert.NotNull(state); + var cursor = state!.Cursor; + var pendingDocuments = ExtractGuidList(cursor, "pendingDocuments"); + Assert.Equal(2, pendingDocuments.Count); + + // Re-seed responses to simulate unchanged fetch + SeedIndex(handler); + SeedDetail(handler); + await connector.FetchAsync(provider, CancellationToken.None); + + state = await stateRepository.TryGetAsync(VndrAdobeConnectorPlugin.SourceName, CancellationToken.None); + Assert.NotNull(state); + cursor = state!.Cursor; + var afterPending = ExtractGuidList(cursor, "pendingDocuments"); + Assert.Equal(pendingDocuments.OrderBy(static id => id), afterPending.OrderBy(static id => id)); + + var fetchCache = cursor.TryGetValue("fetchCache", out var fetchCacheValue) && fetchCacheValue is BsonDocument cacheDoc + ? cacheDoc.Elements.Select(static e => e.Name).ToArray() + : Array.Empty(); + Assert.Contains("https://helpx.adobe.com/security/products/acrobat/apsb25-85.html", fetchCache); + Assert.Contains("https://helpx.adobe.com/security/products/premiere_pro/apsb25-87.html", fetchCache); + } + + [Fact] + public async Task Parse_ProducesDtoAndClearsPendingDocuments() + { + var handler = new CannedHttpMessageHandler(); + await using var provider = await BuildServiceProviderAsync(handler); + SeedIndex(handler); + SeedDetail(handler); + + var connector = provider.GetRequiredService(); + await connector.FetchAsync(provider, CancellationToken.None); + await connector.ParseAsync(provider, CancellationToken.None); + await connector.MapAsync(provider, CancellationToken.None); + + var documentStore = provider.GetRequiredService(); + var dtoStore = provider.GetRequiredService(); + var advisoryStore = provider.GetRequiredService(); + var psirtStore = provider.GetRequiredService(); + var stateRepository = provider.GetRequiredService(); + + var document = await documentStore.FindBySourceAndUriAsync( + VndrAdobeConnectorPlugin.SourceName, + "https://helpx.adobe.com/security/products/acrobat/apsb25-85.html", + CancellationToken.None); + + Assert.NotNull(document); + + var dtoRecord = await dtoStore.FindByDocumentIdAsync(document!.Id, CancellationToken.None); + Assert.NotNull(dtoRecord); + Assert.Equal("adobe.bulletin.v1", dtoRecord!.SchemaVersion); + var payload = dtoRecord.Payload; + Assert.Equal("APSB25-85", payload.GetValue("advisoryId").AsString); + Assert.Equal("https://helpx.adobe.com/security/products/acrobat/apsb25-85.html", payload.GetValue("detailUrl").AsString); + + var products = payload.GetValue("products").AsBsonArray + .Select(static value => value.AsBsonDocument) + .ToArray(); + Assert.NotEmpty(products); + var acrobatWindowsProduct = Assert.Single( + products, + static doc => string.Equals(doc.GetValue("product").AsString, "Acrobat DC", StringComparison.Ordinal) + && string.Equals(doc.GetValue("platform").AsString, "Windows", StringComparison.Ordinal)); + Assert.Equal("25.001.20672 and earlier", acrobatWindowsProduct.GetValue("affectedVersion").AsString); + Assert.Equal("25.001.20680", acrobatWindowsProduct.GetValue("updatedVersion").AsString); + + var acrobatMacProduct = Assert.Single( + products, + static doc => string.Equals(doc.GetValue("product").AsString, "Acrobat DC", StringComparison.Ordinal) + && string.Equals(doc.GetValue("platform").AsString, "macOS", StringComparison.Ordinal)); + Assert.Equal("25.001.20668 and earlier", acrobatMacProduct.GetValue("affectedVersion").AsString); + Assert.Equal("25.001.20678", acrobatMacProduct.GetValue("updatedVersion").AsString); + + var state = await stateRepository.TryGetAsync(VndrAdobeConnectorPlugin.SourceName, CancellationToken.None); + Assert.NotNull(state); + var cursor = state!.Cursor; + Assert.True(!cursor.TryGetValue("pendingDocuments", out _) + || cursor.GetValue("pendingDocuments").AsBsonArray.Count == 0); + Assert.True(!cursor.TryGetValue("pendingMappings", out _) + || cursor.GetValue("pendingMappings").AsBsonArray.Count == 0); + + var advisories = await advisoryStore.GetRecentAsync(5, CancellationToken.None); + Assert.Equal(2, advisories.Count); + + var acrobatAdvisory = advisories.Single(a => a.AdvisoryKey == "APSB25-85"); + Assert.Contains("APSB25-85", acrobatAdvisory.Aliases); + Assert.Equal( + acrobatAdvisory.References.Select(static r => r.Url).Distinct(StringComparer.OrdinalIgnoreCase).Count(), + acrobatAdvisory.References.Length); + var acrobatWindowsPackage = Assert.Single( + acrobatAdvisory.AffectedPackages, + pkg => string.Equals(pkg.Identifier, "Acrobat DC", StringComparison.Ordinal) + && string.Equals(pkg.Platform, "Windows", StringComparison.Ordinal)); + var acrobatWindowsRange = Assert.Single(acrobatWindowsPackage.VersionRanges); + Assert.Equal("vendor", acrobatWindowsRange.RangeKind); + Assert.Equal("25.001.20680", acrobatWindowsRange.FixedVersion); + Assert.Equal("25.001.20672", acrobatWindowsRange.LastAffectedVersion); + Assert.NotNull(acrobatWindowsRange.Primitives); + var windowsExtensions = acrobatWindowsRange.Primitives!.VendorExtensions; + Assert.NotNull(windowsExtensions); + Assert.True(windowsExtensions!.TryGetValue("adobe.affected.raw", out var rawAffectedWin)); + Assert.Equal("25.001.20672 and earlier", rawAffectedWin); + Assert.True(windowsExtensions.TryGetValue("adobe.updated.raw", out var rawUpdatedWin)); + Assert.Equal("25.001.20680", rawUpdatedWin); + Assert.Contains( + AffectedPackageStatusCatalog.Fixed, + acrobatWindowsPackage.Statuses.Select(static status => status.Status)); + + var acrobatMacPackage = Assert.Single( + acrobatAdvisory.AffectedPackages, + pkg => string.Equals(pkg.Identifier, "Acrobat DC", StringComparison.Ordinal) + && string.Equals(pkg.Platform, "macOS", StringComparison.Ordinal)); + var acrobatMacRange = Assert.Single(acrobatMacPackage.VersionRanges); + Assert.Equal("vendor", acrobatMacRange.RangeKind); + Assert.Equal("25.001.20678", acrobatMacRange.FixedVersion); + Assert.Equal("25.001.20668", acrobatMacRange.LastAffectedVersion); + Assert.NotNull(acrobatMacRange.Primitives); + var macExtensions = acrobatMacRange.Primitives!.VendorExtensions; + Assert.NotNull(macExtensions); + Assert.True(macExtensions!.TryGetValue("adobe.affected.raw", out var rawAffectedMac)); + Assert.Equal("25.001.20668 and earlier", rawAffectedMac); + Assert.True(macExtensions.TryGetValue("adobe.updated.raw", out var rawUpdatedMac)); + Assert.Equal("25.001.20678", rawUpdatedMac); + Assert.Contains( + AffectedPackageStatusCatalog.Fixed, + acrobatMacPackage.Statuses.Select(static status => status.Status)); + + var premiereAdvisory = advisories.Single(a => a.AdvisoryKey == "APSB25-87"); + Assert.Contains("APSB25-87", premiereAdvisory.Aliases); + Assert.Equal( + premiereAdvisory.References.Select(static r => r.Url).Distinct(StringComparer.OrdinalIgnoreCase).Count(), + premiereAdvisory.References.Length); + var premiereWindowsPackage = Assert.Single( + premiereAdvisory.AffectedPackages, + pkg => string.Equals(pkg.Identifier, "Premiere Pro", StringComparison.Ordinal) + && string.Equals(pkg.Platform, "Windows", StringComparison.Ordinal)); + var premiereWindowsRange = Assert.Single(premiereWindowsPackage.VersionRanges); + Assert.Equal("24.6", premiereWindowsRange.FixedVersion); + Assert.Equal("24.5", premiereWindowsRange.LastAffectedVersion); + Assert.NotNull(premiereWindowsRange.Primitives); + var premiereWindowsExtensions = premiereWindowsRange.Primitives!.VendorExtensions; + Assert.NotNull(premiereWindowsExtensions); + Assert.True(premiereWindowsExtensions!.TryGetValue("adobe.priority", out var premierePriorityWin)); + Assert.Equal("Priority 3", premierePriorityWin); + Assert.Contains( + AffectedPackageStatusCatalog.Fixed, + premiereWindowsPackage.Statuses.Select(static status => status.Status)); + + var premiereMacPackage = Assert.Single( + premiereAdvisory.AffectedPackages, + pkg => string.Equals(pkg.Identifier, "Premiere Pro", StringComparison.Ordinal) + && string.Equals(pkg.Platform, "macOS", StringComparison.Ordinal)); + var premiereMacRange = Assert.Single(premiereMacPackage.VersionRanges); + Assert.Equal("24.6", premiereMacRange.FixedVersion); + Assert.Equal("24.5", premiereMacRange.LastAffectedVersion); + Assert.NotNull(premiereMacRange.Primitives); + var premiereMacExtensions = premiereMacRange.Primitives!.VendorExtensions; + Assert.NotNull(premiereMacExtensions); + Assert.True(premiereMacExtensions!.TryGetValue("adobe.priority", out var premierePriorityMac)); + Assert.Equal("Priority 3", premierePriorityMac); + Assert.Contains( + AffectedPackageStatusCatalog.Fixed, + premiereMacPackage.Statuses.Select(static status => status.Status)); + + var ordered = advisories.OrderBy(static a => a.AdvisoryKey, StringComparer.Ordinal).ToArray(); + var snapshot = SnapshotSerializer.ToSnapshot(ordered); + var expected = ReadFixture("adobe-advisories.snapshot.json"); + var normalizedSnapshot = NormalizeLineEndings(snapshot); + var normalizedExpected = NormalizeLineEndings(expected); + if (!string.Equals(normalizedExpected, normalizedSnapshot, StringComparison.Ordinal)) + { + var actualPath = Path.Combine(AppContext.BaseDirectory, "Source", "Vndr", "Adobe", "Fixtures", "adobe-advisories.actual.json"); + Directory.CreateDirectory(Path.GetDirectoryName(actualPath)!); + File.WriteAllText(actualPath, snapshot); + } + + Assert.Equal(normalizedExpected, normalizedSnapshot); + + var flagsCollection = _fixture.Database.GetCollection("psirt_flags"); + var rawFlags = await flagsCollection.Find(Builders.Filter.Empty).ToListAsync(); + Assert.NotEmpty(rawFlags); + + var flagRecord = rawFlags.Single(doc => doc["_id"].AsString == "APSB25-87"); + Assert.Equal("Adobe", flagRecord["vendor"].AsString); + } + + [Fact] + public async Task Fetch_WithNotModifiedResponses_KeepsDocumentsMapped() + { + var handler = new CannedHttpMessageHandler(); + await using var provider = await BuildServiceProviderAsync(handler); + SeedIndex(handler); + SeedDetail(handler); + + var connector = provider.GetRequiredService(); + await connector.FetchAsync(provider, CancellationToken.None); + await connector.ParseAsync(provider, CancellationToken.None); + await connector.MapAsync(provider, CancellationToken.None); + + var documentStore = provider.GetRequiredService(); + var acrobatDoc = await documentStore.FindBySourceAndUriAsync( + VndrAdobeConnectorPlugin.SourceName, + "https://helpx.adobe.com/security/products/acrobat/apsb25-85.html", + CancellationToken.None); + Assert.NotNull(acrobatDoc); + Assert.Equal(DocumentStatuses.Mapped, acrobatDoc!.Status); + + var premiereDoc = await documentStore.FindBySourceAndUriAsync( + VndrAdobeConnectorPlugin.SourceName, + "https://helpx.adobe.com/security/products/premiere_pro/apsb25-87.html", + CancellationToken.None); + Assert.NotNull(premiereDoc); + Assert.Equal(DocumentStatuses.Mapped, premiereDoc!.Status); + + SeedIndex(handler); + SeedDetailNotModified(handler); + + await connector.FetchAsync(provider, CancellationToken.None); + + acrobatDoc = await documentStore.FindBySourceAndUriAsync( + VndrAdobeConnectorPlugin.SourceName, + "https://helpx.adobe.com/security/products/acrobat/apsb25-85.html", + CancellationToken.None); + Assert.NotNull(acrobatDoc); + Assert.Equal(DocumentStatuses.Mapped, acrobatDoc!.Status); + + premiereDoc = await documentStore.FindBySourceAndUriAsync( + VndrAdobeConnectorPlugin.SourceName, + "https://helpx.adobe.com/security/products/premiere_pro/apsb25-87.html", + CancellationToken.None); + Assert.NotNull(premiereDoc); + Assert.Equal(DocumentStatuses.Mapped, premiereDoc!.Status); + + var stateRepository = provider.GetRequiredService(); + var state = await stateRepository.TryGetAsync(VndrAdobeConnectorPlugin.SourceName, CancellationToken.None); + Assert.NotNull(state); + Assert.True(state!.Cursor.TryGetValue("pendingDocuments", out var pendingDocs) && pendingDocs.AsBsonArray.Count == 0); + Assert.True(state.Cursor.TryGetValue("pendingMappings", out var pendingMap) && pendingMap.AsBsonArray.Count == 0); + } + + private async Task BuildServiceProviderAsync(CannedHttpMessageHandler handler) + { + await _fixture.Client.DropDatabaseAsync(_fixture.Database.DatabaseNamespace.DatabaseName); + + var services = new ServiceCollection(); + services.AddLogging(builder => builder.AddProvider(NullLoggerProvider.Instance)); + services.AddSingleton(_timeProvider); + services.AddSingleton(handler); + + services.AddMongoStorage(options => + { + options.ConnectionString = _fixture.Runner.ConnectionString; + options.DatabaseName = _fixture.Database.DatabaseNamespace.DatabaseName; + options.CommandTimeout = TimeSpan.FromSeconds(5); + }); + + services.AddSourceCommon(); + services.AddAdobeConnector(opts => + { + opts.IndexUri = new Uri("https://helpx.adobe.com/security/security-bulletin.html"); + opts.InitialBackfill = TimeSpan.FromDays(30); + opts.WindowOverlap = TimeSpan.FromDays(2); + }); + + services.Configure(AdobeOptions.HttpClientName, builderOptions => + { + builderOptions.HttpMessageHandlerBuilderActions.Add(builder => + { + builder.PrimaryHandler = handler; + }); + }); + + var provider = services.BuildServiceProvider(); + var bootstrapper = provider.GetRequiredService(); + await bootstrapper.InitializeAsync(CancellationToken.None); + return provider; + } + + private static void SeedIndex(CannedHttpMessageHandler handler) + { + var indexUri = new Uri("https://helpx.adobe.com/security/security-bulletin.html"); + var indexHtml = ReadFixture("adobe-index.html"); + handler.AddTextResponse(indexUri, indexHtml, "text/html"); + } + + private static void SeedDetail(CannedHttpMessageHandler handler) + { + AddDetailResponse( + handler, + new Uri("https://helpx.adobe.com/security/products/acrobat/apsb25-85.html"), + "adobe-detail-apsb25-85.html", + "\"apsb25-85\""); + + AddDetailResponse( + handler, + new Uri("https://helpx.adobe.com/security/products/premiere_pro/apsb25-87.html"), + "adobe-detail-apsb25-87.html", + "\"apsb25-87\""); + } + + private static void SeedDetailNotModified(CannedHttpMessageHandler handler) + { + AddNotModifiedResponse( + handler, + new Uri("https://helpx.adobe.com/security/products/acrobat/apsb25-85.html"), + "\"apsb25-85\""); + + AddNotModifiedResponse( + handler, + new Uri("https://helpx.adobe.com/security/products/premiere_pro/apsb25-87.html"), + "\"apsb25-87\""); + } + + private static void AddDetailResponse(CannedHttpMessageHandler handler, Uri uri, string fixture, string? etag) + { + handler.AddResponse(uri, () => + { + var response = new HttpResponseMessage(HttpStatusCode.OK) + { + Content = new StringContent(ReadFixture(fixture), Encoding.UTF8, "text/html"), + }; + + if (!string.IsNullOrEmpty(etag)) + { + response.Headers.ETag = new EntityTagHeaderValue(etag); + } + + return response; + }); + } + + private static void AddNotModifiedResponse(CannedHttpMessageHandler handler, Uri uri, string? etag) + { + handler.AddResponse(uri, () => + { + var response = new HttpResponseMessage(HttpStatusCode.NotModified); + if (!string.IsNullOrEmpty(etag)) + { + response.Headers.ETag = new EntityTagHeaderValue(etag); + } + + return response; + }); + } + + private static List ExtractGuidList(BsonDocument cursor, string field) + { + if (!cursor.TryGetValue(field, out var value) || value is not BsonArray array) + { + return new List(); + } + + var list = new List(array.Count); + foreach (var element in array) + { + if (Guid.TryParse(element.AsString, out var guid)) + { + list.Add(guid); + } + } + return list; + } + + private static string ReadFixture(string name) + { + var candidate = Path.Combine(AppContext.BaseDirectory, "Adobe", "Fixtures", name); + if (!File.Exists(candidate)) + { + candidate = Path.Combine(AppContext.BaseDirectory, "Source", "Vndr", "Adobe", "Fixtures", name); + } + + return File.ReadAllText(candidate); + } + + private static string NormalizeLineEndings(string value) + => value.Replace("\r\n", "\n", StringComparison.Ordinal); + + public Task InitializeAsync() => Task.CompletedTask; + + public Task DisposeAsync() => Task.CompletedTask; +} diff --git a/src/StellaOps.Feedser.Source.Vndr.Adobe.Tests/Adobe/Fixtures/adobe-advisories.snapshot.json b/src/StellaOps.Feedser.Source.Vndr.Adobe.Tests/Adobe/Fixtures/adobe-advisories.snapshot.json new file mode 100644 index 00000000..4ea9a81d --- /dev/null +++ b/src/StellaOps.Feedser.Source.Vndr.Adobe.Tests/Adobe/Fixtures/adobe-advisories.snapshot.json @@ -0,0 +1,466 @@ +[ + { + "advisoryKey": "APSB25-85", + "affectedPackages": [ + { + "identifier": "Acrobat DC", + "platform": "Windows", + "provenance": [ + { + "fieldMask": [], + "kind": "affected", + "recordedAt": "2025-09-10T00:00:00+00:00", + "source": "vndr-adobe", + "value": "Acrobat DC:Windows" + } + ], + "statuses": [ + { + "provenance": { + "fieldMask": [], + "kind": "affected", + "recordedAt": "2025-09-10T00:00:00+00:00", + "source": "vndr-adobe", + "value": "Acrobat DC:Windows" + }, + "status": "fixed" + } + ], + "type": "vendor", + "versionRanges": [ + { + "fixedVersion": "25.001.20680", + "introducedVersion": null, + "lastAffectedVersion": "25.001.20672", + "primitives": { + "evr": null, + "hasVendorExtensions": true, + "nevra": null, + "semVer": { + "constraintExpression": null, + "fixed": "25.1.20680", + "fixedInclusive": false, + "introduced": null, + "introducedInclusive": true, + "lastAffected": "25.1.20672", + "lastAffectedInclusive": true + }, + "vendorExtensions": { + "adobe.track": "Continuous", + "adobe.platform": "Windows", + "adobe.affected.raw": "25.001.20672 and earlier", + "adobe.updated.raw": "25.001.20680", + "adobe.priority": "Priority 2", + "adobe.availability": "Available" + } + }, + "provenance": { + "fieldMask": [], + "kind": "range", + "recordedAt": "2025-09-10T00:00:00+00:00", + "source": "vndr-adobe", + "value": "Acrobat DC:Windows" + }, + "rangeExpression": "25.001.20672 and earlier", + "rangeKind": "vendor" + } + ] + }, + { + "identifier": "Acrobat DC", + "platform": "macOS", + "provenance": [ + { + "fieldMask": [], + "kind": "affected", + "recordedAt": "2025-09-10T00:00:00+00:00", + "source": "vndr-adobe", + "value": "Acrobat DC:macOS" + } + ], + "statuses": [ + { + "provenance": { + "fieldMask": [], + "kind": "affected", + "recordedAt": "2025-09-10T00:00:00+00:00", + "source": "vndr-adobe", + "value": "Acrobat DC:macOS" + }, + "status": "fixed" + } + ], + "type": "vendor", + "versionRanges": [ + { + "fixedVersion": "25.001.20678", + "introducedVersion": null, + "lastAffectedVersion": "25.001.20668", + "primitives": { + "evr": null, + "hasVendorExtensions": true, + "nevra": null, + "semVer": { + "constraintExpression": null, + "fixed": "25.1.20678", + "fixedInclusive": false, + "introduced": null, + "introducedInclusive": true, + "lastAffected": "25.1.20668", + "lastAffectedInclusive": true + }, + "vendorExtensions": { + "adobe.track": "Continuous", + "adobe.platform": "macOS", + "adobe.affected.raw": "25.001.20668 and earlier", + "adobe.updated.raw": "25.001.20678", + "adobe.priority": "Priority 2", + "adobe.availability": "Available" + } + }, + "provenance": { + "fieldMask": [], + "kind": "range", + "recordedAt": "2025-09-10T00:00:00+00:00", + "source": "vndr-adobe", + "value": "Acrobat DC:macOS" + }, + "rangeExpression": "25.001.20668 and earlier", + "rangeKind": "vendor" + } + ] + }, + { + "identifier": "Acrobat Reader DC", + "platform": "Windows", + "provenance": [ + { + "fieldMask": [], + "kind": "affected", + "recordedAt": "2025-09-10T00:00:00+00:00", + "source": "vndr-adobe", + "value": "Acrobat Reader DC:Windows" + } + ], + "statuses": [ + { + "provenance": { + "fieldMask": [], + "kind": "affected", + "recordedAt": "2025-09-10T00:00:00+00:00", + "source": "vndr-adobe", + "value": "Acrobat Reader DC:Windows" + }, + "status": "fixed" + } + ], + "type": "vendor", + "versionRanges": [ + { + "fixedVersion": "25.001.20680", + "introducedVersion": null, + "lastAffectedVersion": "25.001.20672", + "primitives": { + "evr": null, + "hasVendorExtensions": true, + "nevra": null, + "semVer": { + "constraintExpression": null, + "fixed": "25.1.20680", + "fixedInclusive": false, + "introduced": null, + "introducedInclusive": true, + "lastAffected": "25.1.20672", + "lastAffectedInclusive": true + }, + "vendorExtensions": { + "adobe.track": "Continuous", + "adobe.platform": "Windows", + "adobe.affected.raw": "25.001.20672 and earlier", + "adobe.updated.raw": "25.001.20680", + "adobe.priority": "Priority 2", + "adobe.availability": "Available" + } + }, + "provenance": { + "fieldMask": [], + "kind": "range", + "recordedAt": "2025-09-10T00:00:00+00:00", + "source": "vndr-adobe", + "value": "Acrobat Reader DC:Windows" + }, + "rangeExpression": "25.001.20672 and earlier", + "rangeKind": "vendor" + } + ] + }, + { + "identifier": "Acrobat Reader DC", + "platform": "macOS", + "provenance": [ + { + "fieldMask": [], + "kind": "affected", + "recordedAt": "2025-09-10T00:00:00+00:00", + "source": "vndr-adobe", + "value": "Acrobat Reader DC:macOS" + } + ], + "statuses": [ + { + "provenance": { + "fieldMask": [], + "kind": "affected", + "recordedAt": "2025-09-10T00:00:00+00:00", + "source": "vndr-adobe", + "value": "Acrobat Reader DC:macOS" + }, + "status": "fixed" + } + ], + "type": "vendor", + "versionRanges": [ + { + "fixedVersion": "25.001.20678", + "introducedVersion": null, + "lastAffectedVersion": "25.001.20668", + "primitives": { + "evr": null, + "hasVendorExtensions": true, + "nevra": null, + "semVer": { + "constraintExpression": null, + "fixed": "25.1.20678", + "fixedInclusive": false, + "introduced": null, + "introducedInclusive": true, + "lastAffected": "25.1.20668", + "lastAffectedInclusive": true + }, + "vendorExtensions": { + "adobe.track": "Continuous", + "adobe.platform": "macOS", + "adobe.affected.raw": "25.001.20668 and earlier", + "adobe.updated.raw": "25.001.20678", + "adobe.priority": "Priority 2", + "adobe.availability": "Available" + } + }, + "provenance": { + "fieldMask": [], + "kind": "range", + "recordedAt": "2025-09-10T00:00:00+00:00", + "source": "vndr-adobe", + "value": "Acrobat Reader DC:macOS" + }, + "rangeExpression": "25.001.20668 and earlier", + "rangeKind": "vendor" + } + ] + } + ], + "aliases": [ + "APSB25-85" + ], + "cvssMetrics": [], + "exploitKnown": false, + "language": "en", + "modified": null, + "provenance": [ + { + "fieldMask": [], + "kind": "parser", + "recordedAt": "2025-09-10T00:00:00+00:00", + "source": "vndr-adobe", + "value": "APSB25-85" + } + ], + "published": "2025-09-09T00:00:00+00:00", + "references": [ + { + "kind": "advisory", + "provenance": { + "fieldMask": [], + "kind": "parser", + "recordedAt": "2025-09-10T00:00:00+00:00", + "source": "vndr-adobe", + "value": "APSB25-85" + }, + "sourceTag": "adobe-psirt", + "summary": "Date published: September 9, 2025", + "url": "https://helpx.adobe.com/security/products/acrobat/apsb25-85.html" + } + ], + "severity": null, + "summary": "Date published: September 9, 2025", + "title": "APSB25-85: Security update available for Adobe Acrobat Reader" + }, + { + "advisoryKey": "APSB25-87", + "affectedPackages": [ + { + "identifier": "Premiere Pro", + "platform": "Windows", + "provenance": [ + { + "fieldMask": [], + "kind": "affected", + "recordedAt": "2025-09-10T00:00:00+00:00", + "source": "vndr-adobe", + "value": "Premiere Pro:Windows" + } + ], + "statuses": [ + { + "provenance": { + "fieldMask": [], + "kind": "affected", + "recordedAt": "2025-09-10T00:00:00+00:00", + "source": "vndr-adobe", + "value": "Premiere Pro:Windows" + }, + "status": "fixed" + } + ], + "type": "vendor", + "versionRanges": [ + { + "fixedVersion": "24.6", + "introducedVersion": null, + "lastAffectedVersion": "24.5", + "primitives": { + "evr": null, + "hasVendorExtensions": true, + "nevra": null, + "semVer": { + "constraintExpression": null, + "fixed": "24.6", + "fixedInclusive": false, + "introduced": null, + "introducedInclusive": true, + "lastAffected": "24.5", + "lastAffectedInclusive": true + }, + "vendorExtensions": { + "adobe.track": "Quarterly", + "adobe.platform": "Windows", + "adobe.affected.raw": "24.5 and earlier", + "adobe.updated.raw": "24.6", + "adobe.priority": "Priority 3", + "adobe.availability": "Available" + } + }, + "provenance": { + "fieldMask": [], + "kind": "range", + "recordedAt": "2025-09-10T00:00:00+00:00", + "source": "vndr-adobe", + "value": "Premiere Pro:Windows" + }, + "rangeExpression": "24.5 and earlier", + "rangeKind": "vendor" + } + ] + }, + { + "identifier": "Premiere Pro", + "platform": "macOS", + "provenance": [ + { + "fieldMask": [], + "kind": "affected", + "recordedAt": "2025-09-10T00:00:00+00:00", + "source": "vndr-adobe", + "value": "Premiere Pro:macOS" + } + ], + "statuses": [ + { + "provenance": { + "fieldMask": [], + "kind": "affected", + "recordedAt": "2025-09-10T00:00:00+00:00", + "source": "vndr-adobe", + "value": "Premiere Pro:macOS" + }, + "status": "fixed" + } + ], + "type": "vendor", + "versionRanges": [ + { + "fixedVersion": "24.6", + "introducedVersion": null, + "lastAffectedVersion": "24.5", + "primitives": { + "evr": null, + "hasVendorExtensions": true, + "nevra": null, + "semVer": { + "constraintExpression": null, + "fixed": "24.6", + "fixedInclusive": false, + "introduced": null, + "introducedInclusive": true, + "lastAffected": "24.5", + "lastAffectedInclusive": true + }, + "vendorExtensions": { + "adobe.track": "Quarterly", + "adobe.platform": "macOS", + "adobe.affected.raw": "24.5 and earlier", + "adobe.updated.raw": "24.6", + "adobe.priority": "Priority 3", + "adobe.availability": "Available" + } + }, + "provenance": { + "fieldMask": [], + "kind": "range", + "recordedAt": "2025-09-10T00:00:00+00:00", + "source": "vndr-adobe", + "value": "Premiere Pro:macOS" + }, + "rangeExpression": "24.5 and earlier", + "rangeKind": "vendor" + } + ] + } + ], + "aliases": [ + "APSB25-87" + ], + "cvssMetrics": [], + "exploitKnown": false, + "language": "en", + "modified": null, + "provenance": [ + { + "fieldMask": [], + "kind": "parser", + "recordedAt": "2025-09-10T00:00:00+00:00", + "source": "vndr-adobe", + "value": "APSB25-87" + } + ], + "published": "2025-09-08T00:00:00+00:00", + "references": [ + { + "kind": "advisory", + "provenance": { + "fieldMask": [], + "kind": "parser", + "recordedAt": "2025-09-10T00:00:00+00:00", + "source": "vndr-adobe", + "value": "APSB25-87" + }, + "sourceTag": "adobe-psirt", + "summary": "Date published: September 8, 2025", + "url": "https://helpx.adobe.com/security/products/premiere_pro/apsb25-87.html" + } + ], + "severity": null, + "summary": "Date published: September 8, 2025", + "title": "APSB25-87: Security update available for Adobe Premiere Pro" + } +] \ No newline at end of file diff --git a/src/StellaOps.Feedser.Source.Vndr.Adobe.Tests/Adobe/Fixtures/adobe-detail-apsb25-85.html b/src/StellaOps.Feedser.Source.Vndr.Adobe.Tests/Adobe/Fixtures/adobe-detail-apsb25-85.html new file mode 100644 index 00000000..90278190 --- /dev/null +++ b/src/StellaOps.Feedser.Source.Vndr.Adobe.Tests/Adobe/Fixtures/adobe-detail-apsb25-85.html @@ -0,0 +1,72 @@ + + + + APSB25-85 + + +

    APSB25-85: Security update available for Adobe Acrobat Reader

    +

    Date published: September 9, 2025

    + +

    Affected Versions

    + + + + + + + + + + + + + + + + + + + +
    ProductTrackAffected VersionsPlatform
    Acrobat DCContinuous +

    Win - 25.001.20672 and earlier

    +

    Mac - 25.001.20668 and earlier

    +     		Windows & macOS
    Acrobat Reader DCContinuous +

    Win - 25.001.20672 and earlier

    +

    Mac - 25.001.20668 and earlier

    +     		Windows & macOS
    + +

    Updated Versions

    + + + + + + + + + + + + + + + + + + + + + + + + + +
    ProductTrackUpdated VersionsPlatformPriorityAvailability
    Acrobat DCContinuous +

    Win - 25.001.20680

    +

    Mac - 25.001.20678

    +     		Windows & macOSPriority 2Available
    Acrobat Reader DCContinuous +

    Win - 25.001.20680

    +

    Mac - 25.001.20678

    +     		Windows & macOSPriority 2Available
    + + diff --git a/src/StellaOps.Feedser.Source.Vndr.Adobe.Tests/Adobe/Fixtures/adobe-detail-apsb25-87.html b/src/StellaOps.Feedser.Source.Vndr.Adobe.Tests/Adobe/Fixtures/adobe-detail-apsb25-87.html new file mode 100644 index 00000000..5d4d13da --- /dev/null +++ b/src/StellaOps.Feedser.Source.Vndr.Adobe.Tests/Adobe/Fixtures/adobe-detail-apsb25-87.html @@ -0,0 +1,52 @@ + + + + APSB25-87 + + +

    APSB25-87: Security update available for Adobe Premiere Pro

    +

    Date published: September 8, 2025

    + +

    Affected Versions

    + + + + + + + + + + + + + +
    ProductTrackAffected VersionsPlatform
    Premiere ProQuarterly +

    Win - 24.5 and earlier

    +

    Mac - 24.5 and earlier

    +     		Windows & macOS
    + +

    Updated Versions

    + + + + + + + + + + + + + + + + + +
    ProductTrackUpdated VersionsPlatformPriorityAvailability
    Premiere ProQuarterly +

    Win - 24.6

    +

    Mac - 24.6

    +     		Windows & macOSPriority 3Available
    + + diff --git a/src/StellaOps.Feedser.Source.Vndr.Adobe.Tests/Adobe/Fixtures/adobe-index.html b/src/StellaOps.Feedser.Source.Vndr.Adobe.Tests/Adobe/Fixtures/adobe-index.html new file mode 100644 index 00000000..7bae415d --- /dev/null +++ b/src/StellaOps.Feedser.Source.Vndr.Adobe.Tests/Adobe/Fixtures/adobe-index.html @@ -0,0 +1,17 @@ + + + + + + + + + + + + + + +
    APSB25-85: Security update available for Adobe Acrobat ReaderSeptember 9, 2025
    APSB25-87: Security update available for Adobe Premiere Pro09/08/2025
    + + diff --git a/src/StellaOps.Feedser.Source.Vndr.Adobe.Tests/StellaOps.Feedser.Source.Vndr.Adobe.Tests.csproj b/src/StellaOps.Feedser.Source.Vndr.Adobe.Tests/StellaOps.Feedser.Source.Vndr.Adobe.Tests.csproj new file mode 100644 index 00000000..2446f91a --- /dev/null +++ b/src/StellaOps.Feedser.Source.Vndr.Adobe.Tests/StellaOps.Feedser.Source.Vndr.Adobe.Tests.csproj @@ -0,0 +1,17 @@ + + + net10.0 + enable + enable + + + + + + + + + + + + diff --git a/src/StellaOps.Feedser.Source.Vndr.Adobe/AGENTS.md b/src/StellaOps.Feedser.Source.Vndr.Adobe/AGENTS.md new file mode 100644 index 00000000..227fc8b1 --- /dev/null +++ b/src/StellaOps.Feedser.Source.Vndr.Adobe/AGENTS.md @@ -0,0 +1,28 @@ +# AGENTS +## Role +Adobe PSIRT connector ingesting APSB/APA advisories; authoritative for Adobe products; emits psirt_flags and affected ranges; establishes PSIRT precedence over registry or distro data for Adobe software. +## Scope +- Discover and fetch APSB/APA index and detail pages; follow product links as needed; window by advisory ID/date. +- Validate HTML or JSON; normalize titles, CVE lists, product components, fixed versions/builds; capture mitigation notes and KBs. +- Persist raw docs with sha256 and headers; maintain source_state cursors; ensure idempotent mapping. +## Participants +- Source.Common (HTTP, HTML parsing, retries/backoff, validators). +- Storage.Mongo (document, dto, advisory, alias, affected, reference, psirt_flags, source_state). +- Models (canonical Advisory/Affected/Provenance). +- Core/WebService (jobs: source:adobe:fetch|parse|map). +- Merge engine (later) to apply PSIRT override policy for Adobe packages. +## Interfaces & contracts +- Aliases include APSB-YYYY-XX (and APA-* when present) plus CVE ids. +- Affected entries capture Vendor=Adobe, Product/component names, Type=vendor, Identifier stable (for example product slug), Versions with fixed/fixedBy where available. +- References typed: advisory, patch, mitigation, release notes; URLs normalized and deduped. +- Provenance.method="parser"; value carries advisory id and URL; recordedAt=fetch time. +## In/Out of scope +In: PSIRT ingestion, aliases, affected plus fixedBy, psirt_flags, watermark/resume. +Out: signing, package artifact downloads, non-Adobe product truth. +## Observability & security expectations +- Metrics: SourceDiagnostics produces `feedser.source.http.*` counters/histograms tagged `feedser.source=adobe`; operators filter on that tag to monitor fetch counts, parse failures, map affected counts, and cursor movement without bespoke metric names. +- Logs: advisory ids, product counts, extraction timings; hosts allowlisted; no secret logging. +## Tests +- Author and review coverage in `../StellaOps.Feedser.Source.Vndr.Adobe.Tests`. +- Shared fixtures (e.g., `MongoIntegrationFixture`, `ConnectorTestHarness`) live in `../StellaOps.Feedser.Testing`. +- Keep fixtures deterministic; match new cases to real-world advisories or regression scenarios. diff --git a/src/StellaOps.Feedser.Source.Vndr.Adobe/AdobeConnector.cs b/src/StellaOps.Feedser.Source.Vndr.Adobe/AdobeConnector.cs new file mode 100644 index 00000000..c5a793fd --- /dev/null +++ b/src/StellaOps.Feedser.Source.Vndr.Adobe/AdobeConnector.cs @@ -0,0 +1,720 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Text.Json; +using System.Text.Json.Serialization; +using System.Text.RegularExpressions; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Options; +using Json.Schema; +using MongoDB.Bson; +using StellaOps.Feedser.Source.Common; +using StellaOps.Feedser.Source.Common.Fetch; +using StellaOps.Feedser.Source.Common.Json; +using StellaOps.Feedser.Source.Common.Packages; +using StellaOps.Feedser.Source.Vndr.Adobe.Configuration; +using StellaOps.Feedser.Source.Vndr.Adobe.Internal; +using StellaOps.Feedser.Storage.Mongo; +using StellaOps.Feedser.Storage.Mongo.Advisories; +using StellaOps.Feedser.Storage.Mongo.Documents; +using StellaOps.Feedser.Storage.Mongo.Dtos; +using StellaOps.Feedser.Storage.Mongo.PsirtFlags; +using StellaOps.Feedser.Models; +using StellaOps.Plugin; + +namespace StellaOps.Feedser.Source.Vndr.Adobe; + +public sealed class AdobeConnector : IFeedConnector +{ + private readonly SourceFetchService _fetchService; + private readonly RawDocumentStorage _rawDocumentStorage; + private readonly IDocumentStore _documentStore; + private readonly IDtoStore _dtoStore; + private readonly IAdvisoryStore _advisoryStore; + private readonly ISourceStateRepository _stateRepository; + private readonly IPsirtFlagStore _psirtFlagStore; + private readonly IJsonSchemaValidator _schemaValidator; + private readonly AdobeOptions _options; + private readonly TimeProvider _timeProvider; + private readonly IHttpClientFactory _httpClientFactory; + private readonly AdobeDiagnostics _diagnostics; + private readonly ILogger _logger; + + private static readonly JsonSchema Schema = AdobeSchemaProvider.Schema; + private static readonly JsonSerializerOptions SerializerOptions = new() + { + PropertyNamingPolicy = JsonNamingPolicy.CamelCase, + DefaultIgnoreCondition = JsonIgnoreCondition.WhenWritingNull, + }; + + public AdobeConnector( + SourceFetchService fetchService, + RawDocumentStorage rawDocumentStorage, + IDocumentStore documentStore, + IDtoStore dtoStore, + IAdvisoryStore advisoryStore, + ISourceStateRepository stateRepository, + IPsirtFlagStore psirtFlagStore, + IJsonSchemaValidator schemaValidator, + IOptions options, + TimeProvider? timeProvider, + IHttpClientFactory httpClientFactory, + AdobeDiagnostics diagnostics, + ILogger logger) + { + _fetchService = fetchService ?? throw new ArgumentNullException(nameof(fetchService)); + _rawDocumentStorage = rawDocumentStorage ?? throw new ArgumentNullException(nameof(rawDocumentStorage)); + _documentStore = documentStore ?? throw new ArgumentNullException(nameof(documentStore)); + _dtoStore = dtoStore ?? throw new ArgumentNullException(nameof(dtoStore)); + _advisoryStore = advisoryStore ?? throw new ArgumentNullException(nameof(advisoryStore)); + _stateRepository = stateRepository ?? throw new ArgumentNullException(nameof(stateRepository)); + _psirtFlagStore = psirtFlagStore ?? throw new ArgumentNullException(nameof(psirtFlagStore)); + _schemaValidator = schemaValidator ?? throw new ArgumentNullException(nameof(schemaValidator)); + _options = options?.Value ?? throw new ArgumentNullException(nameof(options)); + _options.Validate(); + _timeProvider = timeProvider ?? TimeProvider.System; + _httpClientFactory = httpClientFactory ?? throw new ArgumentNullException(nameof(httpClientFactory)); + _diagnostics = diagnostics ?? throw new ArgumentNullException(nameof(diagnostics)); + _logger = logger ?? throw new ArgumentNullException(nameof(logger)); + } + + private static IReadOnlyList BuildStatuses(AdobeProductEntry product, AdvisoryProvenance provenance) + { + if (!TryResolveAvailabilityStatus(product.Availability, out var status)) + { + return Array.Empty(); + } + + return new[] { new AffectedPackageStatus(status, provenance) }; + } + + private static bool TryResolveAvailabilityStatus(string? availability, out string status) + { + status = string.Empty; + if (string.IsNullOrWhiteSpace(availability)) + { + return false; + } + + var trimmed = availability.Trim(); + + if (AffectedPackageStatusCatalog.TryNormalize(trimmed, out var normalized)) + { + status = normalized; + return true; + } + + var token = SanitizeStatusToken(trimmed); + if (token.Length == 0) + { + return false; + } + + if (AvailabilityStatusMap.TryGetValue(token, out var mapped)) + { + status = mapped; + return true; + } + + return false; + } + + private static string SanitizeStatusToken(string value) + { + var buffer = new char[value.Length]; + var index = 0; + + foreach (var ch in value) + { + if (char.IsLetterOrDigit(ch)) + { + buffer[index++] = char.ToLowerInvariant(ch); + } + } + + return index == 0 ? string.Empty : new string(buffer, 0, index); + } + + private static readonly Dictionary AvailabilityStatusMap = new(StringComparer.Ordinal) + { + ["available"] = AffectedPackageStatusCatalog.Fixed, + ["availabletoday"] = AffectedPackageStatusCatalog.Fixed, + ["availablenow"] = AffectedPackageStatusCatalog.Fixed, + ["updateavailable"] = AffectedPackageStatusCatalog.Fixed, + ["patchavailable"] = AffectedPackageStatusCatalog.Fixed, + ["fixavailable"] = AffectedPackageStatusCatalog.Fixed, + ["mitigationavailable"] = AffectedPackageStatusCatalog.Mitigated, + ["workaroundavailable"] = AffectedPackageStatusCatalog.Mitigated, + ["mitigationprovided"] = AffectedPackageStatusCatalog.Mitigated, + ["workaroundprovided"] = AffectedPackageStatusCatalog.Mitigated, + ["planned"] = AffectedPackageStatusCatalog.Pending, + ["updateplanned"] = AffectedPackageStatusCatalog.Pending, + ["plannedupdate"] = AffectedPackageStatusCatalog.Pending, + ["scheduled"] = AffectedPackageStatusCatalog.Pending, + ["scheduledupdate"] = AffectedPackageStatusCatalog.Pending, + ["pendingavailability"] = AffectedPackageStatusCatalog.Pending, + ["pendingupdate"] = AffectedPackageStatusCatalog.Pending, + ["pendingfix"] = AffectedPackageStatusCatalog.Pending, + ["notavailable"] = AffectedPackageStatusCatalog.Unknown, + ["unavailable"] = AffectedPackageStatusCatalog.Unknown, + ["notcurrentlyavailable"] = AffectedPackageStatusCatalog.Unknown, + ["notapplicable"] = AffectedPackageStatusCatalog.NotApplicable, + }; + + private AffectedVersionRange? BuildVersionRange(AdobeProductEntry product, DateTimeOffset recordedAt) + { + if (string.IsNullOrWhiteSpace(product.AffectedVersion) && string.IsNullOrWhiteSpace(product.UpdatedVersion)) + { + return null; + } + + var key = string.IsNullOrWhiteSpace(product.Platform) + ? product.Product + : $"{product.Product}:{product.Platform}"; + + var provenance = new AdvisoryProvenance(SourceName, "range", key, recordedAt); + + var extensions = new Dictionary(StringComparer.Ordinal); + AddExtension(extensions, "adobe.track", product.Track); + AddExtension(extensions, "adobe.platform", product.Platform); + AddExtension(extensions, "adobe.affected.raw", product.AffectedVersion); + AddExtension(extensions, "adobe.updated.raw", product.UpdatedVersion); + AddExtension(extensions, "adobe.priority", product.Priority); + AddExtension(extensions, "adobe.availability", product.Availability); + + var lastAffected = ExtractVersionNumber(product.AffectedVersion); + var fixedVersion = ExtractVersionNumber(product.UpdatedVersion); + + var primitives = BuildRangePrimitives(lastAffected, fixedVersion, extensions); + + return new AffectedVersionRange( + rangeKind: "vendor", + introducedVersion: null, + fixedVersion: fixedVersion, + lastAffectedVersion: lastAffected, + rangeExpression: product.AffectedVersion ?? product.UpdatedVersion, + provenance: provenance, + primitives: primitives); + } + + private static RangePrimitives? BuildRangePrimitives(string? lastAffected, string? fixedVersion, Dictionary extensions) + { + var semVer = BuildSemVerPrimitive(lastAffected, fixedVersion); + + if (semVer is null && extensions.Count == 0) + { + return null; + } + + return new RangePrimitives(semVer, null, null, extensions.Count == 0 ? null : extensions); + } + + private static SemVerPrimitive? BuildSemVerPrimitive(string? lastAffected, string? fixedVersion) + { + var fixedNormalized = NormalizeSemVer(fixedVersion); + var lastNormalized = NormalizeSemVer(lastAffected); + + if (fixedNormalized is null && lastNormalized is null) + { + return null; + } + + return new SemVerPrimitive( + Introduced: null, + IntroducedInclusive: true, + Fixed: fixedNormalized, + FixedInclusive: false, + LastAffected: lastNormalized, + LastAffectedInclusive: true, + ConstraintExpression: null); + } + + private static string? NormalizeSemVer(string? value) + { + if (string.IsNullOrWhiteSpace(value)) + { + return null; + } + + var trimmed = value.Trim(); + if (PackageCoordinateHelper.TryParseSemVer(trimmed, out _, out var normalized) && !string.IsNullOrWhiteSpace(normalized)) + { + return normalized; + } + + if (Version.TryParse(trimmed, out var parsed)) + { + if (parsed.Build >= 0 && parsed.Revision >= 0) + { + return $"{parsed.Major}.{parsed.Minor}.{parsed.Build}.{parsed.Revision}"; + } + + if (parsed.Build >= 0) + { + return $"{parsed.Major}.{parsed.Minor}.{parsed.Build}"; + } + + return $"{parsed.Major}.{parsed.Minor}"; + } + + return null; + } + + private static string? ExtractVersionNumber(string? text) + { + if (string.IsNullOrWhiteSpace(text)) + { + return null; + } + + var match = VersionPattern.Match(text); + return match.Success ? match.Value : null; + } + + private static void AddExtension(IDictionary extensions, string key, string? value) + { + if (!string.IsNullOrWhiteSpace(value)) + { + extensions[key] = value.Trim(); + } + } + + private static readonly Regex VersionPattern = new("\\d+(?:\\.\\d+)+", RegexOptions.Compiled); + + public string SourceName => VndrAdobeConnectorPlugin.SourceName; + + public async Task FetchAsync(IServiceProvider services, CancellationToken cancellationToken) + { + var cursor = await GetCursorAsync(cancellationToken).ConfigureAwait(false); + var now = _timeProvider.GetUtcNow(); + var backfillStart = now - _options.InitialBackfill; + var windowStart = cursor.LastPublished.HasValue + ? cursor.LastPublished.Value - _options.WindowOverlap + : backfillStart; + if (windowStart < backfillStart) + { + windowStart = backfillStart; + } + + var maxPublished = cursor.LastPublished; + var pendingDocuments = cursor.PendingDocuments.ToList(); + var pendingMappings = cursor.PendingMappings.ToList(); + var fetchCache = cursor.FetchCache is null + ? new Dictionary(StringComparer.Ordinal) + : new Dictionary(cursor.FetchCache, StringComparer.Ordinal); + var touchedResources = new HashSet(StringComparer.Ordinal); + + var collectedEntries = new Dictionary(StringComparer.OrdinalIgnoreCase); + + foreach (var indexUri in EnumerateIndexUris()) + { + _diagnostics.FetchAttempt(); + string? html = null; + try + { + var client = _httpClientFactory.CreateClient(AdobeOptions.HttpClientName); + using var response = await client.GetAsync(indexUri, cancellationToken).ConfigureAwait(false); + response.EnsureSuccessStatusCode(); + html = await response.Content.ReadAsStringAsync(cancellationToken).ConfigureAwait(false); + } + catch (Exception ex) + { + _diagnostics.FetchFailure(); + _logger.LogError(ex, "Failed to download Adobe index page {Uri}", indexUri); + continue; + } + + if (string.IsNullOrEmpty(html)) + { + continue; + } + + IReadOnlyCollection entries; + try + { + entries = AdobeIndexParser.Parse(html, indexUri); + } + catch (Exception ex) + { + _logger.LogError(ex, "Failed to parse Adobe index page {Uri}", indexUri); + _diagnostics.FetchFailure(); + continue; + } + + foreach (var entry in entries) + { + if (entry.PublishedUtc < windowStart) + { + continue; + } + + if (!collectedEntries.TryGetValue(entry.AdvisoryId, out var existing) || entry.PublishedUtc > existing.PublishedUtc) + { + collectedEntries[entry.AdvisoryId] = entry; + } + } + } + + foreach (var entry in collectedEntries.Values.OrderBy(static e => e.PublishedUtc)) + { + if (!maxPublished.HasValue || entry.PublishedUtc > maxPublished) + { + maxPublished = entry.PublishedUtc; + } + + var cacheKey = entry.DetailUri.ToString(); + touchedResources.Add(cacheKey); + + var metadata = new Dictionary(StringComparer.Ordinal) + { + ["advisoryId"] = entry.AdvisoryId, + ["published"] = entry.PublishedUtc.ToString("O"), + ["title"] = entry.Title ?? string.Empty, + }; + + try + { + var result = await _fetchService.FetchAsync( + new SourceFetchRequest(AdobeOptions.HttpClientName, SourceName, entry.DetailUri) + { + Metadata = metadata, + AcceptHeaders = new[] { "text/html", "application/xhtml+xml", "text/plain;q=0.5" }, + }, + cancellationToken).ConfigureAwait(false); + + if (!result.IsSuccess || result.Document is null) + { + continue; + } + + if (cursor.TryGetFetchCache(cacheKey, out var cached) + && string.Equals(cached.Sha256, result.Document.Sha256, StringComparison.OrdinalIgnoreCase)) + { + _diagnostics.FetchUnchanged(); + fetchCache[cacheKey] = new AdobeFetchCacheEntry(result.Document.Sha256); + await _documentStore.UpdateStatusAsync(result.Document.Id, DocumentStatuses.Mapped, cancellationToken).ConfigureAwait(false); + continue; + } + + _diagnostics.FetchDocument(); + fetchCache[cacheKey] = new AdobeFetchCacheEntry(result.Document.Sha256); + + if (!pendingDocuments.Contains(result.Document.Id)) + { + pendingDocuments.Add(result.Document.Id); + } + } + catch (Exception ex) + { + _diagnostics.FetchFailure(); + _logger.LogError(ex, "Failed to fetch Adobe advisory {AdvisoryId} ({Uri})", entry.AdvisoryId, entry.DetailUri); + await _stateRepository.MarkFailureAsync(SourceName, now, TimeSpan.FromMinutes(10), ex.Message, cancellationToken).ConfigureAwait(false); + throw; + } + } + + foreach (var key in fetchCache.Keys.ToList()) + { + if (!touchedResources.Contains(key)) + { + fetchCache.Remove(key); + } + } + + var updatedCursor = cursor + .WithPendingDocuments(pendingDocuments) + .WithPendingMappings(pendingMappings) + .WithLastPublished(maxPublished) + .WithFetchCache(fetchCache); + + await UpdateCursorAsync(updatedCursor, cancellationToken).ConfigureAwait(false); + } + + public async Task ParseAsync(IServiceProvider services, CancellationToken cancellationToken) + { + var cursor = await GetCursorAsync(cancellationToken).ConfigureAwait(false); + if (cursor.PendingDocuments.Count == 0) + { + return; + } + + var pendingDocuments = cursor.PendingDocuments.ToList(); + var pendingMappings = cursor.PendingMappings.ToList(); + + foreach (var documentId in cursor.PendingDocuments) + { + var document = await _documentStore.FindAsync(documentId, cancellationToken).ConfigureAwait(false); + if (document is null) + { + pendingDocuments.Remove(documentId); + pendingMappings.Remove(documentId); + continue; + } + + if (!document.GridFsId.HasValue) + { + _logger.LogWarning("Adobe document {DocumentId} missing GridFS payload", document.Id); + await _documentStore.UpdateStatusAsync(document.Id, DocumentStatuses.Failed, cancellationToken).ConfigureAwait(false); + pendingDocuments.Remove(documentId); + pendingMappings.Remove(documentId); + continue; + } + + AdobeDocumentMetadata metadata; + try + { + metadata = AdobeDocumentMetadata.FromDocument(document); + } + catch (Exception ex) + { + _logger.LogError(ex, "Adobe metadata parse failed for document {DocumentId}", document.Id); + await _documentStore.UpdateStatusAsync(document.Id, DocumentStatuses.Failed, cancellationToken).ConfigureAwait(false); + pendingDocuments.Remove(documentId); + pendingMappings.Remove(documentId); + continue; + } + + AdobeBulletinDto dto; + try + { + var bytes = await _rawDocumentStorage.DownloadAsync(document.GridFsId.Value, cancellationToken).ConfigureAwait(false); + var html = Encoding.UTF8.GetString(bytes); + dto = AdobeDetailParser.Parse(html, metadata); + } + catch (Exception ex) + { + _logger.LogError(ex, "Adobe parse failed for advisory {AdvisoryId} ({Uri})", metadata.AdvisoryId, document.Uri); + await _documentStore.UpdateStatusAsync(document.Id, DocumentStatuses.Failed, cancellationToken).ConfigureAwait(false); + pendingDocuments.Remove(documentId); + pendingMappings.Remove(documentId); + continue; + } + + var json = JsonSerializer.Serialize(dto, SerializerOptions); + using var jsonDocument = JsonDocument.Parse(json); + _schemaValidator.Validate(jsonDocument, Schema, metadata.AdvisoryId); + + var payload = MongoDB.Bson.BsonDocument.Parse(json); + var dtoRecord = new DtoRecord( + Guid.NewGuid(), + document.Id, + SourceName, + "adobe.bulletin.v1", + payload, + _timeProvider.GetUtcNow()); + + await _dtoStore.UpsertAsync(dtoRecord, cancellationToken).ConfigureAwait(false); + await _documentStore.UpdateStatusAsync(document.Id, DocumentStatuses.PendingMap, cancellationToken).ConfigureAwait(false); + + pendingDocuments.Remove(documentId); + if (!pendingMappings.Contains(documentId)) + { + pendingMappings.Add(documentId); + } + } + + var updatedCursor = cursor + .WithPendingDocuments(pendingDocuments) + .WithPendingMappings(pendingMappings); + + await UpdateCursorAsync(updatedCursor, cancellationToken).ConfigureAwait(false); + } + + public async Task MapAsync(IServiceProvider services, CancellationToken cancellationToken) + { + var cursor = await GetCursorAsync(cancellationToken).ConfigureAwait(false); + if (cursor.PendingMappings.Count == 0) + { + return; + } + + var pendingMappings = cursor.PendingMappings.ToList(); + var now = _timeProvider.GetUtcNow(); + + foreach (var documentId in cursor.PendingMappings) + { + var dtoRecord = await _dtoStore.FindByDocumentIdAsync(documentId, cancellationToken).ConfigureAwait(false); + var document = await _documentStore.FindAsync(documentId, cancellationToken).ConfigureAwait(false); + + if (dtoRecord is null || document is null) + { + pendingMappings.Remove(documentId); + continue; + } + + AdobeBulletinDto? dto; + try + { + var json = dtoRecord.Payload.ToJson(new MongoDB.Bson.IO.JsonWriterSettings + { + OutputMode = MongoDB.Bson.IO.JsonOutputMode.RelaxedExtendedJson, + }); + + dto = JsonSerializer.Deserialize(json, SerializerOptions); + } + catch (Exception ex) + { + _logger.LogError(ex, "Adobe DTO deserialization failed for document {DocumentId}", documentId); + await _documentStore.UpdateStatusAsync(document.Id, DocumentStatuses.Failed, cancellationToken).ConfigureAwait(false); + pendingMappings.Remove(documentId); + continue; + } + + if (dto is null) + { + _logger.LogWarning("Adobe DTO payload deserialized as null for document {DocumentId}", documentId); + await _documentStore.UpdateStatusAsync(document.Id, DocumentStatuses.Failed, cancellationToken).ConfigureAwait(false); + pendingMappings.Remove(documentId); + continue; + } + + var advisory = BuildAdvisory(dto, now); + if (!string.IsNullOrWhiteSpace(advisory.AdvisoryKey)) + { + await _advisoryStore.UpsertAsync(advisory, cancellationToken).ConfigureAwait(false); + + var flag = new PsirtFlagRecord( + advisory.AdvisoryKey, + "Adobe", + SourceName, + dto.AdvisoryId, + now); + + await _psirtFlagStore.UpsertAsync(flag, cancellationToken).ConfigureAwait(false); + } + else + { + _logger.LogWarning("Skipping PSIRT flag for advisory with missing key (document {DocumentId})", documentId); + } + + await _documentStore.UpdateStatusAsync(document.Id, DocumentStatuses.Mapped, cancellationToken).ConfigureAwait(false); + + pendingMappings.Remove(documentId); + } + + var updatedCursor = cursor.WithPendingMappings(pendingMappings); + await UpdateCursorAsync(updatedCursor, cancellationToken).ConfigureAwait(false); + } + + private IEnumerable EnumerateIndexUris() + { + yield return _options.IndexUri; + foreach (var uri in _options.AdditionalIndexUris) + { + yield return uri; + } + } + + private async Task GetCursorAsync(CancellationToken cancellationToken) + { + var record = await _stateRepository.TryGetAsync(SourceName, cancellationToken).ConfigureAwait(false); + return AdobeCursor.FromBsonDocument(record?.Cursor); + } + + private async Task UpdateCursorAsync(AdobeCursor cursor, CancellationToken cancellationToken) + { + var updatedAt = _timeProvider.GetUtcNow(); + await _stateRepository.UpdateCursorAsync(SourceName, cursor.ToBsonDocument(), updatedAt, cancellationToken).ConfigureAwait(false); + } + + private Advisory BuildAdvisory(AdobeBulletinDto dto, DateTimeOffset recordedAt) + { + var provenance = new AdvisoryProvenance(SourceName, "parser", dto.AdvisoryId, recordedAt); + + var aliasSet = new HashSet(StringComparer.OrdinalIgnoreCase) + { + dto.AdvisoryId, + }; + foreach (var cve in dto.Cves) + { + if (!string.IsNullOrWhiteSpace(cve)) + { + aliasSet.Add(cve); + } + } + + var comparer = StringComparer.OrdinalIgnoreCase; + var references = new List<(AdvisoryReference Reference, int Priority)> + { + (new AdvisoryReference(dto.DetailUrl, "advisory", "adobe-psirt", dto.Summary, provenance), 0), + }; + + foreach (var cve in dto.Cves) + { + if (string.IsNullOrWhiteSpace(cve)) + { + continue; + } + + var url = $"https://www.cve.org/CVERecord?id={cve}"; + references.Add((new AdvisoryReference(url, "advisory", cve, null, provenance), 1)); + } + + var orderedReferences = references + .GroupBy(tuple => tuple.Reference.Url, comparer) + .Select(group => group + .OrderBy(t => t.Priority) + .ThenBy(t => t.Reference.Kind ?? string.Empty, comparer) + .ThenBy(t => t.Reference.Url, comparer) + .First()) + .OrderBy(t => t.Priority) + .ThenBy(t => t.Reference.Kind ?? string.Empty, comparer) + .ThenBy(t => t.Reference.Url, comparer) + .Select(t => t.Reference) + .ToArray(); + + var affected = dto.Products + .Select(product => BuildPackage(product, recordedAt)) + .ToArray(); + + var aliases = aliasSet + .Where(static alias => !string.IsNullOrWhiteSpace(alias)) + .Select(static alias => alias.Trim()) + .Distinct(StringComparer.OrdinalIgnoreCase) + .OrderBy(static alias => alias, StringComparer.Ordinal) + .ToArray(); + + return new Advisory( + dto.AdvisoryId, + dto.Title, + dto.Summary, + language: "en", + published: dto.Published, + modified: null, + severity: null, + exploitKnown: false, + aliases, + orderedReferences, + affected, + Array.Empty(), + new[] { provenance }); + } + + private AffectedPackage BuildPackage(AdobeProductEntry product, DateTimeOffset recordedAt) + { + var identifier = string.IsNullOrWhiteSpace(product.Product) + ? "Adobe Product" + : product.Product.Trim(); + + var platform = string.IsNullOrWhiteSpace(product.Platform) ? null : product.Platform; + + var provenance = new AdvisoryProvenance( + SourceName, + "affected", + string.IsNullOrWhiteSpace(platform) ? identifier : $"{identifier}:{platform}", + recordedAt); + + var range = BuildVersionRange(product, recordedAt); + var ranges = range is null ? Array.Empty() : new[] { range }; + var statuses = BuildStatuses(product, provenance); + + return new AffectedPackage( + AffectedPackageTypes.Vendor, + identifier, + platform, + ranges, + statuses, + new[] { provenance }); + } +} diff --git a/src/StellaOps.Feedser.Source.Vndr.Adobe/AdobeConnectorPlugin.cs b/src/StellaOps.Feedser.Source.Vndr.Adobe/AdobeConnectorPlugin.cs new file mode 100644 index 00000000..5f3fe442 --- /dev/null +++ b/src/StellaOps.Feedser.Source.Vndr.Adobe/AdobeConnectorPlugin.cs @@ -0,0 +1,21 @@ +using System; +using Microsoft.Extensions.DependencyInjection; +using StellaOps.Plugin; + +namespace StellaOps.Feedser.Source.Vndr.Adobe; + +public sealed class VndrAdobeConnectorPlugin : IConnectorPlugin +{ + public const string SourceName = "vndr-adobe"; + + public string Name => SourceName; + + public bool IsAvailable(IServiceProvider services) + => services.GetService() is not null; + + public IFeedConnector Create(IServiceProvider services) + { + ArgumentNullException.ThrowIfNull(services); + return services.GetRequiredService(); + } +} diff --git a/src/StellaOps.Feedser.Source.Vndr.Adobe/AdobeDiagnostics.cs b/src/StellaOps.Feedser.Source.Vndr.Adobe/AdobeDiagnostics.cs new file mode 100644 index 00000000..a6bcdacf --- /dev/null +++ b/src/StellaOps.Feedser.Source.Vndr.Adobe/AdobeDiagnostics.cs @@ -0,0 +1,49 @@ +using System; +using System.Diagnostics.Metrics; + +namespace StellaOps.Feedser.Source.Vndr.Adobe; + +public sealed class AdobeDiagnostics : IDisposable +{ + public const string MeterName = "StellaOps.Feedser.Source.Vndr.Adobe"; + private static readonly string MeterVersion = "1.0.0"; + + private readonly Meter _meter; + private readonly Counter _fetchAttempts; + private readonly Counter _fetchDocuments; + private readonly Counter _fetchFailures; + private readonly Counter _fetchUnchanged; + + public AdobeDiagnostics() + { + _meter = new Meter(MeterName, MeterVersion); + _fetchAttempts = _meter.CreateCounter( + name: "adobe.fetch.attempts", + unit: "operations", + description: "Number of Adobe index fetch operations."); + _fetchDocuments = _meter.CreateCounter( + name: "adobe.fetch.documents", + unit: "documents", + description: "Number of Adobe advisory documents captured."); + _fetchFailures = _meter.CreateCounter( + name: "adobe.fetch.failures", + unit: "operations", + description: "Number of Adobe fetch failures."); + _fetchUnchanged = _meter.CreateCounter( + name: "adobe.fetch.unchanged", + unit: "documents", + description: "Number of Adobe advisories skipped due to unchanged content."); + } + + public Meter Meter => _meter; + + public void FetchAttempt() => _fetchAttempts.Add(1); + + public void FetchDocument() => _fetchDocuments.Add(1); + + public void FetchFailure() => _fetchFailures.Add(1); + + public void FetchUnchanged() => _fetchUnchanged.Add(1); + + public void Dispose() => _meter.Dispose(); +} diff --git a/src/StellaOps.Feedser.Source.Vndr.Adobe/AdobeServiceCollectionExtensions.cs b/src/StellaOps.Feedser.Source.Vndr.Adobe/AdobeServiceCollectionExtensions.cs new file mode 100644 index 00000000..77f3f581 --- /dev/null +++ b/src/StellaOps.Feedser.Source.Vndr.Adobe/AdobeServiceCollectionExtensions.cs @@ -0,0 +1,38 @@ +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.DependencyInjection.Extensions; +using Microsoft.Extensions.Options; +using StellaOps.Feedser.Source.Common.Http; +using StellaOps.Feedser.Source.Vndr.Adobe.Configuration; + +namespace StellaOps.Feedser.Source.Vndr.Adobe; + +public static class AdobeServiceCollectionExtensions +{ + public static IServiceCollection AddAdobeConnector(this IServiceCollection services, Action configure) + { + ArgumentNullException.ThrowIfNull(services); + ArgumentNullException.ThrowIfNull(configure); + + services.AddOptions() + .Configure(configure) + .PostConfigure(static opts => opts.Validate()); + + services.AddSourceHttpClient(AdobeOptions.HttpClientName, static (sp, options) => + { + var adobeOptions = sp.GetRequiredService>().Value; + options.BaseAddress = adobeOptions.IndexUri; + options.UserAgent = "StellaOps.Feedser.VndrAdobe/1.0"; + options.Timeout = TimeSpan.FromSeconds(20); + options.AllowedHosts.Clear(); + options.AllowedHosts.Add(adobeOptions.IndexUri.Host); + foreach (var additional in adobeOptions.AdditionalIndexUris) + { + options.AllowedHosts.Add(additional.Host); + } + }); + + services.TryAddSingleton(); + services.AddTransient(); + return services; + } +} diff --git a/src/StellaOps.Feedser.Source.Vndr.Adobe/Configuration/AdobeOptions.cs b/src/StellaOps.Feedser.Source.Vndr.Adobe/Configuration/AdobeOptions.cs new file mode 100644 index 00000000..d92a02ee --- /dev/null +++ b/src/StellaOps.Feedser.Source.Vndr.Adobe/Configuration/AdobeOptions.cs @@ -0,0 +1,50 @@ +using System; +using System.Collections.Generic; + +namespace StellaOps.Feedser.Source.Vndr.Adobe.Configuration; + +public sealed class AdobeOptions +{ + public const string HttpClientName = "source-vndr-adobe"; + + public Uri IndexUri { get; set; } = new("https://helpx.adobe.com/security/security-bulletin.html"); + + public List AdditionalIndexUris { get; } = new(); + + public TimeSpan InitialBackfill { get; set; } = TimeSpan.FromDays(90); + + public TimeSpan WindowOverlap { get; set; } = TimeSpan.FromDays(3); + + public int MaxEntriesPerFetch { get; set; } = 100; + + public void Validate() + { + if (IndexUri is null || !IndexUri.IsAbsoluteUri) + { + throw new ArgumentException("IndexUri must be an absolute URI.", nameof(IndexUri)); + } + + foreach (var uri in AdditionalIndexUris) + { + if (uri is null || !uri.IsAbsoluteUri) + { + throw new ArgumentException("Additional index URIs must be absolute.", nameof(AdditionalIndexUris)); + } + } + + if (InitialBackfill <= TimeSpan.Zero) + { + throw new ArgumentException("InitialBackfill must be positive.", nameof(InitialBackfill)); + } + + if (WindowOverlap < TimeSpan.Zero) + { + throw new ArgumentException("WindowOverlap cannot be negative.", nameof(WindowOverlap)); + } + + if (MaxEntriesPerFetch <= 0) + { + throw new ArgumentException("MaxEntriesPerFetch must be positive.", nameof(MaxEntriesPerFetch)); + } + } +} diff --git a/src/StellaOps.Feedser.Source.Vndr.Adobe/Internal/AdobeBulletinDto.cs b/src/StellaOps.Feedser.Source.Vndr.Adobe/Internal/AdobeBulletinDto.cs new file mode 100644 index 00000000..6afd0658 --- /dev/null +++ b/src/StellaOps.Feedser.Source.Vndr.Adobe/Internal/AdobeBulletinDto.cs @@ -0,0 +1,102 @@ +using System; +using System.Collections.Generic; +using System.Linq; + +namespace StellaOps.Feedser.Source.Vndr.Adobe.Internal; + +internal sealed record AdobeBulletinDto( + string AdvisoryId, + string Title, + DateTimeOffset Published, + IReadOnlyList Products, + IReadOnlyList Cves, + string DetailUrl, + string? Summary) +{ + public static AdobeBulletinDto Create( + string advisoryId, + string title, + DateTimeOffset published, + IEnumerable? products, + IEnumerable? cves, + Uri detailUri, + string? summary) + { + ArgumentException.ThrowIfNullOrEmpty(advisoryId); + ArgumentException.ThrowIfNullOrEmpty(title); + ArgumentNullException.ThrowIfNull(detailUri); + + var productList = products? + .Where(static p => !string.IsNullOrWhiteSpace(p.Product)) + .Select(static p => p with { Product = p.Product.Trim() }) + .Distinct(AdobeProductEntryComparer.Instance) + .OrderBy(static p => p.Product, StringComparer.OrdinalIgnoreCase) + .ThenBy(static p => p.Platform, StringComparer.OrdinalIgnoreCase) + .ThenBy(static p => p.Track, StringComparer.OrdinalIgnoreCase) + .ToList() + ?? new List(); + + var cveList = cves?.Where(static c => !string.IsNullOrWhiteSpace(c)) + .Select(static c => c.Trim().ToUpperInvariant()) + .Distinct(StringComparer.Ordinal) + .OrderBy(static c => c, StringComparer.Ordinal) + .ToList() ?? new List(); + + return new AdobeBulletinDto( + advisoryId.ToUpperInvariant(), + title.Trim(), + published.ToUniversalTime(), + productList, + cveList, + detailUri.ToString(), + string.IsNullOrWhiteSpace(summary) ? null : summary.Trim()); + } +} + +internal sealed record AdobeProductEntry( + string Product, + string Track, + string Platform, + string? AffectedVersion, + string? UpdatedVersion, + string? Priority, + string? Availability); + +internal sealed class AdobeProductEntryComparer : IEqualityComparer +{ + public static AdobeProductEntryComparer Instance { get; } = new(); + + public bool Equals(AdobeProductEntry? x, AdobeProductEntry? y) + { + if (ReferenceEquals(x, y)) + { + return true; + } + + if (x is null || y is null) + { + return false; + } + + return string.Equals(x.Product, y.Product, StringComparison.OrdinalIgnoreCase) + && string.Equals(x.Track, y.Track, StringComparison.OrdinalIgnoreCase) + && string.Equals(x.Platform, y.Platform, StringComparison.OrdinalIgnoreCase) + && string.Equals(x.AffectedVersion, y.AffectedVersion, StringComparison.OrdinalIgnoreCase) + && string.Equals(x.UpdatedVersion, y.UpdatedVersion, StringComparison.OrdinalIgnoreCase) + && string.Equals(x.Priority, y.Priority, StringComparison.OrdinalIgnoreCase) + && string.Equals(x.Availability, y.Availability, StringComparison.OrdinalIgnoreCase); + } + + public int GetHashCode(AdobeProductEntry obj) + { + var hash = new HashCode(); + hash.Add(obj.Product, StringComparer.OrdinalIgnoreCase); + hash.Add(obj.Track, StringComparer.OrdinalIgnoreCase); + hash.Add(obj.Platform, StringComparer.OrdinalIgnoreCase); + hash.Add(obj.AffectedVersion, StringComparer.OrdinalIgnoreCase); + hash.Add(obj.UpdatedVersion, StringComparer.OrdinalIgnoreCase); + hash.Add(obj.Priority, StringComparer.OrdinalIgnoreCase); + hash.Add(obj.Availability, StringComparer.OrdinalIgnoreCase); + return hash.ToHashCode(); + } +} diff --git a/src/StellaOps.Feedser.Source.Vndr.Adobe/Internal/AdobeCursor.cs b/src/StellaOps.Feedser.Source.Vndr.Adobe/Internal/AdobeCursor.cs new file mode 100644 index 00000000..e24f17ab --- /dev/null +++ b/src/StellaOps.Feedser.Source.Vndr.Adobe/Internal/AdobeCursor.cs @@ -0,0 +1,168 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using MongoDB.Bson; + +namespace StellaOps.Feedser.Source.Vndr.Adobe.Internal; + +internal sealed record AdobeCursor( + DateTimeOffset? LastPublished, + IReadOnlyCollection PendingDocuments, + IReadOnlyCollection PendingMappings, + IReadOnlyDictionary? FetchCache) +{ + public static AdobeCursor Empty { get; } = new(null, Array.Empty(), Array.Empty(), null); + + public BsonDocument ToBsonDocument() + { + var document = new BsonDocument(); + if (LastPublished.HasValue) + { + document["lastPublished"] = LastPublished.Value.UtcDateTime; + } + + document["pendingDocuments"] = new BsonArray(PendingDocuments.Select(id => id.ToString())); + document["pendingMappings"] = new BsonArray(PendingMappings.Select(id => id.ToString())); + + if (FetchCache is { Count: > 0 }) + { + var cacheDocument = new BsonDocument(); + foreach (var (key, entry) in FetchCache) + { + cacheDocument[key] = entry.ToBson(); + } + + document["fetchCache"] = cacheDocument; + } + + return document; + } + + public static AdobeCursor FromBsonDocument(BsonDocument? document) + { + if (document is null || document.ElementCount == 0) + { + return Empty; + } + + DateTimeOffset? lastPublished = null; + if (document.TryGetValue("lastPublished", out var lastPublishedValue)) + { + lastPublished = ReadDateTime(lastPublishedValue); + } + + var pendingDocuments = ReadGuidArray(document, "pendingDocuments"); + var pendingMappings = ReadGuidArray(document, "pendingMappings"); + var fetchCache = ReadFetchCache(document); + + return new AdobeCursor(lastPublished, pendingDocuments, pendingMappings, fetchCache); + } + + public AdobeCursor WithLastPublished(DateTimeOffset? value) + => this with { LastPublished = value?.ToUniversalTime() }; + + public AdobeCursor WithPendingDocuments(IEnumerable ids) + => this with { PendingDocuments = ids?.Distinct().ToArray() ?? Array.Empty() }; + + public AdobeCursor WithPendingMappings(IEnumerable ids) + => this with { PendingMappings = ids?.Distinct().ToArray() ?? Array.Empty() }; + + public AdobeCursor WithFetchCache(IDictionary? cache) + { + if (cache is null) + { + return this with { FetchCache = null }; + } + + var target = new Dictionary(cache, StringComparer.Ordinal); + return this with { FetchCache = target }; + } + + public bool TryGetFetchCache(string key, out AdobeFetchCacheEntry entry) + { + var cache = FetchCache; + if (cache is null) + { + entry = AdobeFetchCacheEntry.Empty; + return false; + } + + if (cache.TryGetValue(key, out var value) && value is not null) + { + entry = value; + return true; + } + + entry = AdobeFetchCacheEntry.Empty; + return false; + } + + private static DateTimeOffset? ReadDateTime(BsonValue value) + { + return value.BsonType switch + { + BsonType.DateTime => DateTime.SpecifyKind(value.ToUniversalTime(), DateTimeKind.Utc), + BsonType.String when DateTimeOffset.TryParse(value.AsString, out var parsed) => parsed.ToUniversalTime(), + _ => null, + }; + } + + private static IReadOnlyCollection ReadGuidArray(BsonDocument document, string field) + { + if (!document.TryGetValue(field, out var value) || value is not BsonArray array) + { + return Array.Empty(); + } + + var list = new List(array.Count); + foreach (var element in array) + { + if (Guid.TryParse(element.ToString(), out var guid)) + { + list.Add(guid); + } + } + + return list; + } + + private static IReadOnlyDictionary? ReadFetchCache(BsonDocument document) + { + if (!document.TryGetValue("fetchCache", out var value) || value is not BsonDocument cacheDocument) + { + return null; + } + + var dictionary = new Dictionary(StringComparer.Ordinal); + foreach (var element in cacheDocument.Elements) + { + if (element.Value is BsonDocument entryDocument) + { + dictionary[element.Name] = AdobeFetchCacheEntry.FromBson(entryDocument); + } + } + + return dictionary; + } +} + +internal sealed record AdobeFetchCacheEntry(string Sha256) +{ + public static AdobeFetchCacheEntry Empty { get; } = new(string.Empty); + + public BsonDocument ToBson() + { + var document = new BsonDocument + { + ["sha256"] = Sha256, + }; + + return document; + } + + public static AdobeFetchCacheEntry FromBson(BsonDocument document) + { + var sha = document.TryGetValue("sha256", out var shaValue) ? shaValue.AsString : string.Empty; + return new AdobeFetchCacheEntry(sha); + } +} diff --git a/src/StellaOps.Feedser.Source.Vndr.Adobe/Internal/AdobeDetailParser.cs b/src/StellaOps.Feedser.Source.Vndr.Adobe/Internal/AdobeDetailParser.cs new file mode 100644 index 00000000..fdc83bec --- /dev/null +++ b/src/StellaOps.Feedser.Source.Vndr.Adobe/Internal/AdobeDetailParser.cs @@ -0,0 +1,405 @@ +using System; +using System.Collections.Generic; +using System.Globalization; +using System.Linq; +using System.Text.RegularExpressions; +using AngleSharp.Dom; +using AngleSharp.Html.Dom; +using AngleSharp.Html.Parser; + +namespace StellaOps.Feedser.Source.Vndr.Adobe.Internal; + +internal static class AdobeDetailParser +{ + private static readonly HtmlParser Parser = new(); + private static readonly Regex CveRegex = new("CVE-\\d{4}-\\d{4,}", RegexOptions.IgnoreCase | RegexOptions.Compiled); + private static readonly string[] DateMarkers = { "date published", "release date", "published" }; + + public static AdobeBulletinDto Parse(string html, AdobeDocumentMetadata metadata) + { + ArgumentException.ThrowIfNullOrEmpty(html); + ArgumentNullException.ThrowIfNull(metadata); + + using var document = Parser.ParseDocument(html); + var title = metadata.Title ?? document.QuerySelector("h1")?.TextContent?.Trim() ?? metadata.AdvisoryId; + var summary = document.QuerySelector("p")?.TextContent?.Trim(); + + var published = metadata.PublishedUtc ?? TryExtractPublished(document) ?? DateTimeOffset.UtcNow; + + var cves = ExtractCves(document.Body?.TextContent ?? string.Empty); + var products = ExtractProductEntries(title, document); + + return AdobeBulletinDto.Create( + metadata.AdvisoryId, + title, + published, + products, + cves, + metadata.DetailUri, + summary); + } + + private static IReadOnlyList ExtractCves(string text) + { + if (string.IsNullOrWhiteSpace(text)) + { + return Array.Empty(); + } + + var set = new HashSet(StringComparer.OrdinalIgnoreCase); + foreach (Match match in CveRegex.Matches(text)) + { + if (!string.IsNullOrWhiteSpace(match.Value)) + { + set.Add(match.Value.ToUpperInvariant()); + } + } + + return set.Count == 0 ? Array.Empty() : set.OrderBy(static cve => cve, StringComparer.Ordinal).ToArray(); + } + + private static IReadOnlyList ExtractProductEntries(string title, IDocument document) + { + var builders = new Dictionary(AdobeProductKeyComparer.Instance); + + foreach (var builder in ParseAffectedTable(document)) + { + builders[builder.Key] = builder; + } + + foreach (var updated in ParseUpdatedTable(document)) + { + if (builders.TryGetValue(updated.Key, out var builder)) + { + builder.UpdatedVersion ??= updated.UpdatedVersion; + builder.Priority ??= updated.Priority; + builder.Availability ??= updated.Availability; + } + else + { + builders[updated.Key] = updated; + } + } + + if (builders.Count == 0 && !string.IsNullOrWhiteSpace(title)) + { + var fallback = new AdobeProductEntryBuilder( + NormalizeWhitespace(title), + string.Empty, + string.Empty) + { + AffectedVersion = null, + UpdatedVersion = null, + Priority = null, + Availability = null + }; + + builders[fallback.Key] = fallback; + } + + return builders.Values + .Select(static builder => builder.ToEntry()) + .ToList(); + } + + private static IEnumerable ParseAffectedTable(IDocument document) + { + var table = FindTableByHeader(document, "Affected Versions"); + if (table is null) + { + yield break; + } + + foreach (var row in table.Rows.Skip(1)) + { + var cells = row.Cells; + if (cells.Length < 3) + { + continue; + } + + var product = NormalizeWhitespace(cells[0]?.TextContent); + var track = NormalizeWhitespace(cells.ElementAtOrDefault(1)?.TextContent); + var platformText = NormalizeWhitespace(cells.ElementAtOrDefault(3)?.TextContent); + + if (string.IsNullOrWhiteSpace(product)) + { + continue; + } + + var affectedCell = cells[2]; + foreach (var line in ExtractLines(affectedCell)) + { + if (string.IsNullOrWhiteSpace(line)) + { + continue; + } + + var (platform, versionText) = SplitPlatformLine(line, platformText); + var builder = new AdobeProductEntryBuilder(product, track, platform) + { + AffectedVersion = versionText + }; + + yield return builder; + } + } + } + + private static IEnumerable ParseUpdatedTable(IDocument document) + { + var table = FindTableByHeader(document, "Updated Versions"); + if (table is null) + { + yield break; + } + + foreach (var row in table.Rows.Skip(1)) + { + var cells = row.Cells; + if (cells.Length < 3) + { + continue; + } + + var product = NormalizeWhitespace(cells[0]?.TextContent); + var track = NormalizeWhitespace(cells.ElementAtOrDefault(1)?.TextContent); + var platformText = NormalizeWhitespace(cells.ElementAtOrDefault(3)?.TextContent); + var priority = NormalizeWhitespace(cells.ElementAtOrDefault(4)?.TextContent); + var availability = NormalizeWhitespace(cells.ElementAtOrDefault(5)?.TextContent); + + if (string.IsNullOrWhiteSpace(product)) + { + continue; + } + + var updatedCell = cells[2]; + var lines = ExtractLines(updatedCell); + if (lines.Count == 0) + { + lines.Add(updatedCell.TextContent ?? string.Empty); + } + + foreach (var line in lines) + { + if (string.IsNullOrWhiteSpace(line)) + { + continue; + } + + var (platform, versionText) = SplitPlatformLine(line, platformText); + var builder = new AdobeProductEntryBuilder(product, track, platform) + { + UpdatedVersion = versionText, + Priority = priority, + Availability = availability + }; + + yield return builder; + } + } + } + + private static IHtmlTableElement? FindTableByHeader(IDocument document, string headerText) + { + return document + .QuerySelectorAll("table") + .OfType() + .FirstOrDefault(table => table.TextContent.Contains(headerText, StringComparison.OrdinalIgnoreCase)); + } + + private static List ExtractLines(IElement? cell) + { + var lines = new List(); + if (cell is null) + { + return lines; + } + + var paragraphs = cell.QuerySelectorAll("p").Select(static p => p.TextContent).ToArray(); + if (paragraphs.Length > 0) + { + foreach (var paragraph in paragraphs) + { + var normalized = NormalizeWhitespace(paragraph); + if (!string.IsNullOrWhiteSpace(normalized)) + { + lines.Add(normalized); + } + } + + return lines; + } + + var items = cell.QuerySelectorAll("li").Select(static li => li.TextContent).ToArray(); + if (items.Length > 0) + { + foreach (var item in items) + { + var normalized = NormalizeWhitespace(item); + if (!string.IsNullOrWhiteSpace(normalized)) + { + lines.Add(normalized); + } + } + + return lines; + } + + var raw = NormalizeWhitespace(cell.TextContent); + if (!string.IsNullOrWhiteSpace(raw)) + { + lines.AddRange(raw.Split(new[] { '\n' }, StringSplitOptions.TrimEntries | StringSplitOptions.RemoveEmptyEntries)); + } + + return lines; + } + + private static (string Platform, string? Version) SplitPlatformLine(string line, string? fallbackPlatform) + { + var separatorIndex = line.IndexOf('-', StringComparison.Ordinal); + if (separatorIndex > 0 && separatorIndex < line.Length - 1) + { + var prefix = line[..separatorIndex].Trim(); + var versionText = line[(separatorIndex + 1)..].Trim(); + return (NormalizePlatform(prefix) ?? NormalizePlatform(fallbackPlatform) ?? fallbackPlatform ?? string.Empty, versionText); + } + + return (NormalizePlatform(fallbackPlatform) ?? fallbackPlatform ?? string.Empty, line.Trim()); + } + + private static string? NormalizePlatform(string? platform) + { + if (string.IsNullOrWhiteSpace(platform)) + { + return null; + } + + var trimmed = platform.Trim(); + return trimmed.ToLowerInvariant() switch + { + "win" or "windows" => "Windows", + "mac" or "macos" or "mac os" => "macOS", + "windows & macos" or "windows &  macos" => "Windows & macOS", + _ => trimmed + }; + } + + private static DateTimeOffset? TryExtractPublished(IDocument document) + { + var candidates = new List(); + candidates.Add(document.QuerySelector("time")?.GetAttribute("datetime")); + candidates.Add(document.QuerySelector("time")?.TextContent); + + foreach (var marker in DateMarkers) + { + var element = document.All.FirstOrDefault(node => node.TextContent.Contains(marker, StringComparison.OrdinalIgnoreCase)); + if (element is not null) + { + candidates.Add(element.TextContent); + } + } + + foreach (var candidate in candidates) + { + if (TryParseDate(candidate, out var parsed)) + { + return parsed; + } + } + + return null; + } + + private static bool TryParseDate(string? value, out DateTimeOffset result) + { + result = default; + if (string.IsNullOrWhiteSpace(value)) + { + return false; + } + + var trimmed = value.Trim(); + if (DateTimeOffset.TryParse(trimmed, CultureInfo.InvariantCulture, DateTimeStyles.AssumeUniversal | DateTimeStyles.AdjustToUniversal, out result)) + { + result = result.ToUniversalTime(); + return true; + } + + if (DateTime.TryParse(trimmed, CultureInfo.InvariantCulture, DateTimeStyles.None, out var date)) + { + result = new DateTimeOffset(date, TimeSpan.Zero).ToUniversalTime(); + return true; + } + + return false; + } + + private static string NormalizeWhitespace(string? value) + { + if (string.IsNullOrWhiteSpace(value)) + { + return string.Empty; + } + + var sanitized = value ?? string.Empty; + return string.Join(" ", sanitized.Split((char[]?)null, StringSplitOptions.RemoveEmptyEntries)); + } + + private sealed record AdobeProductKey(string Product, string Track, string Platform); + + private sealed class AdobeProductKeyComparer : IEqualityComparer + { + public static AdobeProductKeyComparer Instance { get; } = new(); + + public bool Equals(AdobeProductKey? x, AdobeProductKey? y) + { + if (ReferenceEquals(x, y)) + { + return true; + } + + if (x is null || y is null) + { + return false; + } + + return string.Equals(x.Product, y.Product, StringComparison.OrdinalIgnoreCase) + && string.Equals(x.Track, y.Track, StringComparison.OrdinalIgnoreCase) + && string.Equals(x.Platform, y.Platform, StringComparison.OrdinalIgnoreCase); + } + + public int GetHashCode(AdobeProductKey obj) + { + var hash = new HashCode(); + hash.Add(obj.Product, StringComparer.OrdinalIgnoreCase); + hash.Add(obj.Track, StringComparer.OrdinalIgnoreCase); + hash.Add(obj.Platform, StringComparer.OrdinalIgnoreCase); + return hash.ToHashCode(); + } + } + + private sealed class AdobeProductEntryBuilder + { + public AdobeProductEntryBuilder(string product, string track, string platform) + { + Product = NormalizeWhitespace(product); + Track = NormalizeWhitespace(track); + Platform = NormalizeWhitespace(platform); + } + + public AdobeProductKey Key => new(Product, Track, Platform); + + public string Product { get; } + public string Track { get; } + public string Platform { get; } + + public string? AffectedVersion { get; set; } + public string? UpdatedVersion { get; set; } + public string? Priority { get; set; } + public string? Availability { get; set; } + + public AdobeProductEntry ToEntry() + => new(Product, Track, Platform, AffectedVersion, UpdatedVersion, Priority, Availability); + } +} diff --git a/src/StellaOps.Feedser.Source.Vndr.Adobe/Internal/AdobeDocumentMetadata.cs b/src/StellaOps.Feedser.Source.Vndr.Adobe/Internal/AdobeDocumentMetadata.cs new file mode 100644 index 00000000..616afe37 --- /dev/null +++ b/src/StellaOps.Feedser.Source.Vndr.Adobe/Internal/AdobeDocumentMetadata.cs @@ -0,0 +1,47 @@ +using System; +using System.Collections.Generic; +using StellaOps.Feedser.Storage.Mongo.Documents; + +namespace StellaOps.Feedser.Source.Vndr.Adobe.Internal; + +internal sealed record AdobeDocumentMetadata( + string AdvisoryId, + string? Title, + DateTimeOffset? PublishedUtc, + Uri DetailUri) +{ + private const string AdvisoryIdKey = "advisoryId"; + private const string TitleKey = "title"; + private const string PublishedKey = "published"; + + public static AdobeDocumentMetadata FromDocument(DocumentRecord document) + { + ArgumentNullException.ThrowIfNull(document); + + if (document.Metadata is null) + { + throw new InvalidOperationException("Adobe document metadata is missing."); + } + + var advisoryId = document.Metadata.TryGetValue(AdvisoryIdKey, out var idValue) ? idValue : null; + if (string.IsNullOrWhiteSpace(advisoryId)) + { + throw new InvalidOperationException("Adobe document advisoryId metadata missing."); + } + + var title = document.Metadata.TryGetValue(TitleKey, out var titleValue) ? titleValue : null; + DateTimeOffset? published = null; + if (document.Metadata.TryGetValue(PublishedKey, out var publishedValue) + && DateTimeOffset.TryParse(publishedValue, out var parsedPublished)) + { + published = parsedPublished.ToUniversalTime(); + } + + if (!Uri.TryCreate(document.Uri, UriKind.Absolute, out var detailUri)) + { + throw new InvalidOperationException("Adobe document URI invalid."); + } + + return new AdobeDocumentMetadata(advisoryId.Trim(), string.IsNullOrWhiteSpace(title) ? null : title.Trim(), published, detailUri); + } +} diff --git a/src/StellaOps.Feedser.Source.Vndr.Adobe/Internal/AdobeIndexEntry.cs b/src/StellaOps.Feedser.Source.Vndr.Adobe/Internal/AdobeIndexEntry.cs new file mode 100644 index 00000000..baebbff7 --- /dev/null +++ b/src/StellaOps.Feedser.Source.Vndr.Adobe/Internal/AdobeIndexEntry.cs @@ -0,0 +1,5 @@ +using System; + +namespace StellaOps.Feedser.Source.Vndr.Adobe.Internal; + +internal sealed record AdobeIndexEntry(string AdvisoryId, Uri DetailUri, DateTimeOffset PublishedUtc, string? Title); diff --git a/src/StellaOps.Feedser.Source.Vndr.Adobe/Internal/AdobeIndexParser.cs b/src/StellaOps.Feedser.Source.Vndr.Adobe/Internal/AdobeIndexParser.cs new file mode 100644 index 00000000..738e1044 --- /dev/null +++ b/src/StellaOps.Feedser.Source.Vndr.Adobe/Internal/AdobeIndexParser.cs @@ -0,0 +1,159 @@ +using System; +using System.Collections.Generic; +using System.Globalization; +using System.Linq; +using System.Text.RegularExpressions; +using AngleSharp.Dom; +using AngleSharp.Html.Parser; + +namespace StellaOps.Feedser.Source.Vndr.Adobe.Internal; + +internal static class AdobeIndexParser +{ + private static readonly HtmlParser Parser = new(); + private static readonly Regex AdvisoryIdRegex = new("(APSB|APA)\\d{2}-\\d{2,}", RegexOptions.IgnoreCase | RegexOptions.Compiled); + private static readonly string[] ExplicitFormats = + { + "MMMM d, yyyy", + "MMM d, yyyy", + "M/d/yyyy", + "MM/dd/yyyy", + "yyyy-MM-dd", + }; + + public static IReadOnlyCollection Parse(string html, Uri baseUri) + { + ArgumentNullException.ThrowIfNull(html); + ArgumentNullException.ThrowIfNull(baseUri); + + var document = Parser.ParseDocument(html); + var map = new Dictionary(StringComparer.OrdinalIgnoreCase); + var anchors = document.QuerySelectorAll("a[href]"); + + foreach (var anchor in anchors) + { + var href = anchor.GetAttribute("href"); + if (string.IsNullOrWhiteSpace(href)) + { + continue; + } + + if (!href.Contains("/security/products/", StringComparison.OrdinalIgnoreCase)) + { + continue; + } + + if (!TryExtractAdvisoryId(anchor.TextContent, href, out var advisoryId)) + { + continue; + } + + if (!Uri.TryCreate(baseUri, href, out var detailUri)) + { + continue; + } + + var published = TryResolvePublished(anchor) ?? DateTimeOffset.UtcNow; + var entry = new AdobeIndexEntry(advisoryId.ToUpperInvariant(), detailUri, published, anchor.TextContent?.Trim()); + map[entry.AdvisoryId] = entry; + } + + return map.Values + .OrderBy(static e => e.PublishedUtc) + .ThenBy(static e => e.AdvisoryId, StringComparer.OrdinalIgnoreCase) + .ToArray(); + } + + private static bool TryExtractAdvisoryId(string? text, string href, out string advisoryId) + { + if (!string.IsNullOrWhiteSpace(text)) + { + var match = AdvisoryIdRegex.Match(text); + if (match.Success) + { + advisoryId = match.Value.ToUpperInvariant(); + return true; + } + } + + var hrefMatch = AdvisoryIdRegex.Match(href); + if (hrefMatch.Success) + { + advisoryId = hrefMatch.Value.ToUpperInvariant(); + return true; + } + + advisoryId = string.Empty; + return false; + } + + private static DateTimeOffset? TryResolvePublished(IElement anchor) + { + var row = anchor.Closest("tr"); + if (row is not null) + { + var cells = row.GetElementsByTagName("td"); + if (cells.Length >= 2) + { + for (var idx = 1; idx < cells.Length; idx++) + { + if (TryParseDate(cells[idx].TextContent, out var parsed)) + { + return parsed; + } + } + } + } + + var sibling = anchor.NextElementSibling; + while (sibling is not null) + { + if (TryParseDate(sibling.TextContent, out var parsed)) + { + return parsed; + } + + sibling = sibling.NextElementSibling; + } + + if (TryParseDate(anchor.ParentElement?.TextContent, out var parentDate)) + { + return parentDate; + } + + return null; + } + + private static bool TryParseDate(string? value, out DateTimeOffset result) + { + result = default; + if (string.IsNullOrWhiteSpace(value)) + { + return false; + } + + var trimmed = value.Trim(); + if (DateTimeOffset.TryParse(trimmed, CultureInfo.InvariantCulture, DateTimeStyles.AssumeUniversal | DateTimeStyles.AdjustToUniversal, out result)) + { + return Normalize(ref result); + } + + foreach (var format in ExplicitFormats) + { + if (DateTime.TryParseExact(trimmed, format, CultureInfo.InvariantCulture, DateTimeStyles.None, out var date)) + { + result = new DateTimeOffset(date, TimeSpan.Zero); + return Normalize(ref result); + } + } + + return false; + } + + private static bool Normalize(ref DateTimeOffset value) + { + value = value.ToUniversalTime(); + value = new DateTimeOffset(value.Year, value.Month, value.Day, 0, 0, 0, TimeSpan.Zero); + return true; + } +} diff --git a/src/StellaOps.Feedser.Source.Vndr.Adobe/Internal/AdobeSchemaProvider.cs b/src/StellaOps.Feedser.Source.Vndr.Adobe/Internal/AdobeSchemaProvider.cs new file mode 100644 index 00000000..6f599163 --- /dev/null +++ b/src/StellaOps.Feedser.Source.Vndr.Adobe/Internal/AdobeSchemaProvider.cs @@ -0,0 +1,25 @@ +using System.IO; +using System.Reflection; +using System.Threading; +using Json.Schema; + +namespace StellaOps.Feedser.Source.Vndr.Adobe.Internal; + +internal static class AdobeSchemaProvider +{ + private static readonly Lazy Cached = new(Load, LazyThreadSafetyMode.ExecutionAndPublication); + + public static JsonSchema Schema => Cached.Value; + + private static JsonSchema Load() + { + var assembly = typeof(AdobeSchemaProvider).GetTypeInfo().Assembly; + const string resourceName = "StellaOps.Feedser.Source.Vndr.Adobe.Schemas.adobe-bulletin.schema.json"; + + using var stream = assembly.GetManifestResourceStream(resourceName) + ?? throw new InvalidOperationException($"Embedded schema '{resourceName}' not found."); + using var reader = new StreamReader(stream); + var schemaText = reader.ReadToEnd(); + return JsonSchema.FromText(schemaText); + } +} diff --git a/src/StellaOps.Feedser.Source.Vndr.Adobe/Schemas/adobe-bulletin.schema.json b/src/StellaOps.Feedser.Source.Vndr.Adobe/Schemas/adobe-bulletin.schema.json new file mode 100644 index 00000000..3e972cd8 --- /dev/null +++ b/src/StellaOps.Feedser.Source.Vndr.Adobe/Schemas/adobe-bulletin.schema.json @@ -0,0 +1,78 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://stellaops.example/schemas/adobe-bulletin.schema.json", + "type": "object", + "required": [ + "advisoryId", + "title", + "published", + "products", + "cves", + "detailUrl" + ], + "properties": { + "advisoryId": { + "type": "string", + "minLength": 1 + }, + "title": { + "type": "string", + "minLength": 1 + }, + "published": { + "type": "string", + "format": "date-time" + }, + "products": { + "type": "array", + "items": { + "type": "object", + "required": [ + "product", + "track", + "platform" + ], + "properties": { + "product": { + "type": "string", + "minLength": 1 + }, + "track": { + "type": "string" + }, + "platform": { + "type": "string" + }, + "affectedVersion": { + "type": ["string", "null"] + }, + "updatedVersion": { + "type": ["string", "null"] + }, + "priority": { + "type": ["string", "null"] + }, + "availability": { + "type": ["string", "null"] + } + }, + "additionalProperties": false + } + }, + "cves": { + "type": "array", + "items": { + "type": "string", + "pattern": "^CVE-\\d{4}-\\d{4,}$" + } + }, + "detailUrl": { + "type": "string", + "format": "uri" + }, + "summary": { + "type": ["string", "null"] + } + }, + "additionalProperties": false +} diff --git a/src/StellaOps.Feedser.Source.Vndr.Adobe/StellaOps.Feedser.Source.Vndr.Adobe.csproj b/src/StellaOps.Feedser.Source.Vndr.Adobe/StellaOps.Feedser.Source.Vndr.Adobe.csproj new file mode 100644 index 00000000..5304f8a0 --- /dev/null +++ b/src/StellaOps.Feedser.Source.Vndr.Adobe/StellaOps.Feedser.Source.Vndr.Adobe.csproj @@ -0,0 +1,25 @@ + + + + net10.0 + enable + enable + + + + + + + + + + + + + + + + + + + diff --git a/src/StellaOps.Feedser.Source.Vndr.Adobe/TASKS.md b/src/StellaOps.Feedser.Source.Vndr.Adobe/TASKS.md new file mode 100644 index 00000000..80a21e38 --- /dev/null +++ b/src/StellaOps.Feedser.Source.Vndr.Adobe/TASKS.md @@ -0,0 +1,11 @@ +# TASKS +| Task | Owner(s) | Depends on | Notes | +|---|---|---|---| +|Index discovery and sliding window fetch|BE-Conn-Adobe|Source.Common|DONE — Support backfill; honor robots/ToS.| +|Detail extractor (products/components/fixes)|BE-Conn-Adobe|Source.Common|DONE — Normalizes metadata and CVE/product capture.| +|DTO schema and validation pipeline|BE-Conn-Adobe, QA|Source.Common|DONE — JSON schema enforced during parse.| +|Canonical mapping plus psirt_flags|BE-Conn-Adobe|Models|DONE — Emits canonical advisory and Adobe psirt flag.| +|SourceState plus sha256 short-circuit|BE-Conn-Adobe|Storage.Mongo|DONE — Idempotence guarantee.| +|Golden fixtures and determinism tests|QA|Source.Vndr.Adobe|**DONE** — connector tests assert snapshot determinism for dual advisories.| +|Mark failed parse DTOs|BE-Conn-Adobe|Storage.Mongo|**DONE** — parse failures now mark documents `Failed` and tests cover the path.| +|Reference dedupe & ordering|BE-Conn-Adobe|Models|**DONE** — mapper groups references by URL with deterministic ordering.| diff --git a/src/StellaOps.Feedser.Source.Vndr.Apple.Tests/Apple/AppleConnectorTests.cs b/src/StellaOps.Feedser.Source.Vndr.Apple.Tests/Apple/AppleConnectorTests.cs new file mode 100644 index 00000000..270611ff --- /dev/null +++ b/src/StellaOps.Feedser.Source.Vndr.Apple.Tests/Apple/AppleConnectorTests.cs @@ -0,0 +1,152 @@ +using System.Net; +using System.Net.Http; +using System.Text; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Http; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Logging.Abstractions; +using Microsoft.Extensions.Time.Testing; +using StellaOps.Feedser.Source.Common; +using StellaOps.Feedser.Source.Common.Http; +using StellaOps.Feedser.Source.Common.Fetch; +using StellaOps.Feedser.Source.Common.Testing; +using StellaOps.Feedser.Source.Vndr.Apple; +using StellaOps.Feedser.Storage.Mongo; +using StellaOps.Feedser.Storage.Mongo.Advisories; +using StellaOps.Feedser.Storage.Mongo.PsirtFlags; +using StellaOps.Feedser.Testing; +using Xunit; + +namespace StellaOps.Feedser.Source.Vndr.Apple.Tests; + +[Collection("mongo-fixture")] +public sealed class AppleConnectorTests : IAsyncLifetime +{ + private static readonly Uri IndexUri = new("https://support.example.com/index.json"); + private static readonly Uri DetailBaseUri = new("https://support.example.com/en-us/"); + + private readonly MongoIntegrationFixture _fixture; + private readonly FakeTimeProvider _timeProvider; + + public AppleConnectorTests(MongoIntegrationFixture fixture) + { + _fixture = fixture; + _timeProvider = new FakeTimeProvider(new DateTimeOffset(2025, 10, 10, 0, 0, 0, TimeSpan.Zero)); + } + + [Fact] + public async Task FetchParseMap_EndToEnd_ProducesCanonicalAdvisories() + { + var handler = new CannedHttpMessageHandler(); + SeedIndex(handler); + SeedDetail(handler); + + await using var provider = await BuildServiceProviderAsync(handler); + var connector = provider.GetRequiredService(); + + await connector.FetchAsync(provider, CancellationToken.None); + await connector.ParseAsync(provider, CancellationToken.None); + await connector.MapAsync(provider, CancellationToken.None); + + var advisoryStore = provider.GetRequiredService(); + var advisories = await advisoryStore.GetRecentAsync(10, CancellationToken.None); + Assert.Equal(2, advisories.Count); + + var iosAdvisory = Assert.Single(advisories, advisory => advisory.AdvisoryKey == "HT214108"); + Assert.Contains("CVE-2025-1234", iosAdvisory.Aliases, StringComparer.OrdinalIgnoreCase); + Assert.Contains("CVE-2025-5678", iosAdvisory.Aliases, StringComparer.OrdinalIgnoreCase); + var iosPackage = Assert.Single(iosAdvisory.AffectedPackages); + Assert.Equal("iPhone 15 Pro", iosPackage.Identifier); + var iosRange = Assert.Single(iosPackage.VersionRanges); + Assert.Equal("18.0.1", iosRange.FixedVersion); + Assert.NotNull(iosRange.Primitives); + Assert.Equal("22A123", iosRange.Primitives!.VendorExtensions!["apple.build"]); + + var rsrAdvisory = Assert.Single(advisories, advisory => advisory.AdvisoryKey == "HT215500"); + Assert.Contains("CVE-2025-2468", rsrAdvisory.Aliases, StringComparer.OrdinalIgnoreCase); + + var flagStore = provider.GetRequiredService(); + var rsrFlag = await flagStore.FindAsync("HT215500", CancellationToken.None); + Assert.NotNull(rsrFlag); + } + + public Task InitializeAsync() => Task.CompletedTask; + + public Task DisposeAsync() => Task.CompletedTask; + + private async Task BuildServiceProviderAsync(CannedHttpMessageHandler handler) + { + await _fixture.Client.DropDatabaseAsync(_fixture.Database.DatabaseNamespace.DatabaseName); + + var services = new ServiceCollection(); + services.AddLogging(builder => builder.AddProvider(NullLoggerProvider.Instance)); + services.AddSingleton(_timeProvider); + services.AddSingleton(handler); + + services.AddMongoStorage(options => + { + options.ConnectionString = _fixture.Runner.ConnectionString; + options.DatabaseName = _fixture.Database.DatabaseNamespace.DatabaseName; + options.CommandTimeout = TimeSpan.FromSeconds(5); + }); + + services.AddSourceCommon(); + services.AddAppleConnector(opts => + { + opts.SoftwareLookupUri = IndexUri; + opts.AdvisoryBaseUri = DetailBaseUri; + opts.LocaleSegment = "en-us"; + opts.InitialBackfill = TimeSpan.FromDays(120); + opts.ModifiedTolerance = TimeSpan.FromHours(2); + opts.MaxAdvisoriesPerFetch = 10; + }); + + services.Configure(AppleOptions.HttpClientName, builderOptions => + { + builderOptions.HttpMessageHandlerBuilderActions.Add(builder => + { + builder.PrimaryHandler = handler; + }); + }); + + var provider = services.BuildServiceProvider(); + var bootstrapper = provider.GetRequiredService(); + await bootstrapper.InitializeAsync(CancellationToken.None); + return provider; + } + + private static void SeedIndex(CannedHttpMessageHandler handler) + { + handler.AddJsonResponse(IndexUri, ReadFixture("index.json")); + } + + private static void SeedDetail(CannedHttpMessageHandler handler) + { + AddHtmlResponse(handler, new Uri(DetailBaseUri, "HT214108"), "ht214108.html"); + AddHtmlResponse(handler, new Uri(DetailBaseUri, "HT215500"), "ht215500.html"); + } + + private static void AddHtmlResponse(CannedHttpMessageHandler handler, Uri uri, string fixture) + { + handler.AddResponse(uri, () => + { + var content = ReadFixture(fixture); + return new HttpResponseMessage(HttpStatusCode.OK) + { + Content = new StringContent(content, Encoding.UTF8, "text/html"), + }; + }); + } + + private static string ReadFixture(string name) + { + var path = Path.Combine( + AppContext.BaseDirectory, + "Source", + "Vndr", + "Apple", + "Fixtures", + name); + return File.ReadAllText(path); + } +} diff --git a/src/StellaOps.Feedser.Source.Vndr.Apple.Tests/Apple/AppleFixtureManager.cs b/src/StellaOps.Feedser.Source.Vndr.Apple.Tests/Apple/AppleFixtureManager.cs new file mode 100644 index 00000000..ff68cab1 --- /dev/null +++ b/src/StellaOps.Feedser.Source.Vndr.Apple.Tests/Apple/AppleFixtureManager.cs @@ -0,0 +1,288 @@ +using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using System.Net.Http; +using System.Net.Http.Headers; +using System.Text; +using System.Text.Encodings.Web; +using System.Text.Json; +using System.Text.Json.Serialization; +using System.Threading; +using System.Threading.Tasks; +using StellaOps.Feedser.Source.Vndr.Apple.Internal; + +namespace StellaOps.Feedser.Source.Vndr.Apple.Tests.Apple; + +internal static class AppleFixtureManager +{ + private const string UpdateEnvVar = "UPDATE_APPLE_FIXTURES"; + private static readonly JsonSerializerOptions SerializerOptions = new(JsonSerializerDefaults.Web) + { + WriteIndented = true, + Encoder = JavaScriptEncoder.UnsafeRelaxedJsonEscaping, + DefaultIgnoreCondition = JsonIgnoreCondition.WhenWritingNull, + }; + + private static readonly AppleFixtureDefinition[] Definitions = + { + new( + "125326", + new Uri("https://support.apple.com/en-us/125326"), + Products: new[] + { + new AppleFixtureProduct("iOS", "iPhone 16 Pro", "26.0.1", "24A341"), + new AppleFixtureProduct("iOS", "iPhone 16 Pro", "26.0.1 (a)", "24A341a"), + new AppleFixtureProduct("iPadOS", "iPad Pro (M4)", "26", "24B120"), + }), + new( + "125328", + new Uri("https://support.apple.com/en-us/125328"), + Products: new[] + { + new AppleFixtureProduct("macOS", "MacBook Pro (M4)", "26.0.1", "26A123"), + new AppleFixtureProduct("macOS", "Mac Studio", "26", "26A120b"), + }), + new( + "106355", + new Uri("https://support.apple.com/en-us/106355"), + ForceRapidSecurityResponse: true, + Products: new[] + { + new AppleFixtureProduct("macOS Ventura", "macOS Ventura", string.Empty, "22F400"), + new AppleFixtureProduct("macOS Ventura", "macOS Ventura (Intel)", string.Empty, "22F400a"), + }), + }; + + private static readonly Lazy UpdateTask = new( + () => UpdateFixturesAsync(CancellationToken.None), + LazyThreadSafetyMode.ExecutionAndPublication); + + public static IReadOnlyList Fixtures => Definitions; + + public static Task EnsureUpdatedAsync(CancellationToken cancellationToken = default) + => ShouldUpdateFixtures() + ? UpdateTask.Value + : Task.CompletedTask; + + public static string ReadFixture(string name) + { + var path = Path.Combine(AppContext.BaseDirectory, "Source", "Vndr", "Apple", "Fixtures", name); + return File.ReadAllText(path); + } + + public static AppleDetailDto ReadExpectedDto(string articleId) + { + var json = ReadFixture($"{articleId}.expected.json"); + return JsonSerializer.Deserialize(json, SerializerOptions) + ?? throw new InvalidOperationException($"Unable to deserialize expected DTO for {articleId}."); + } + + private static bool ShouldUpdateFixtures() + { + var value = Environment.GetEnvironmentVariable(UpdateEnvVar)?.Trim(); + return string.Equals(value, "1", StringComparison.Ordinal) + || string.Equals(value, "true", StringComparison.OrdinalIgnoreCase); + } + + private static async Task UpdateFixturesAsync(CancellationToken cancellationToken) + { + var handler = new HttpClientHandler + { + AutomaticDecompression = System.Net.DecompressionMethods.All, + ServerCertificateCustomValidationCallback = HttpClientHandler.DangerousAcceptAnyServerCertificateValidator, + }; + + using var httpClient = new HttpClient(handler); + httpClient.DefaultRequestHeaders.UserAgent.Add(new ProductInfoHeaderValue("StellaOpsFixtureUpdater", "1.0")); + httpClient.Timeout = TimeSpan.FromSeconds(30); + + var indexEntries = new List(Definitions.Length); + + foreach (var definition in Definitions) + { + cancellationToken.ThrowIfCancellationRequested(); + + Console.WriteLine($"[AppleFixtures] Refreshing {definition.ArticleId} from {definition.DetailUri}"); + + var html = await httpClient.GetStringAsync(definition.DetailUri, cancellationToken).ConfigureAwait(false); + var entryProducts = definition.Products? + .Select(product => new AppleIndexProduct( + product.Platform, + product.Name, + product.Version, + product.Build)) + .ToArray() ?? Array.Empty(); + + var entry = new AppleIndexEntry( + UpdateId: definition.ArticleId, + ArticleId: definition.ArticleId, + Title: definition.ArticleId, + PostingDate: DateTimeOffset.UtcNow, + DetailUri: definition.DetailUri, + Products: entryProducts, + IsRapidSecurityResponse: definition.ForceRapidSecurityResponse ?? false); + + var dto = AppleDetailParser.Parse(html, entry); + var sanitizedHtml = BuildSanitizedHtml(dto); + + WriteFixture(definition.HtmlFixtureName, sanitizedHtml); + WriteFixture(definition.ExpectedFixtureName, JsonSerializer.Serialize(dto, SerializerOptions)); + + indexEntries.Add(new + { + id = definition.ArticleId, + articleId = definition.ArticleId, + title = dto.Title, + postingDate = dto.Published.ToUniversalTime().ToString("O"), + detailUrl = definition.DetailUri.ToString(), + rapidSecurityResponse = definition.ForceRapidSecurityResponse + ?? dto.Title.Contains("Rapid Security Response", StringComparison.OrdinalIgnoreCase), + products = dto.Affected + .OrderBy(p => p.Name, StringComparer.OrdinalIgnoreCase) + .Select(p => new + { + platform = p.Platform, + name = p.Name, + version = p.Version, + build = p.Build, + }) + .ToArray(), + }); + } + + var indexDocument = new { updates = indexEntries }; + WriteFixture("index.json", JsonSerializer.Serialize(indexDocument, SerializerOptions)); + } + + private static string BuildSanitizedHtml(AppleDetailDto dto) + { + var builder = new StringBuilder(); + builder.AppendLine(""); + builder.AppendLine(""); + builder.AppendLine(""); + builder.AppendLine("
    "); + + builder.AppendLine($"

    {Escape(dto.Title)}

    "); + + if (!string.IsNullOrWhiteSpace(dto.Summary)) + { + builder.AppendLine($"

    {Escape(dto.Summary)}

    "); + } + + builder.AppendLine($" "); + + if (dto.Updated.HasValue) + { + builder.AppendLine($" "); + } + + if (dto.CveIds.Count > 0) + { + builder.AppendLine("
    "); + builder.AppendLine("

    Security Issues

    "); + builder.AppendLine("
      "); + foreach (var cve in dto.CveIds.OrderBy(id => id, StringComparer.OrdinalIgnoreCase)) + { + builder.AppendLine($"
    • {Escape(cve)}
      • "); + } + builder.AppendLine("
    "); + builder.AppendLine("
    "); + } + + if (dto.Affected.Count > 0) + { + builder.AppendLine(" "); + builder.AppendLine(" "); + foreach (var product in dto.Affected + .OrderBy(p => p.Platform ?? string.Empty, StringComparer.OrdinalIgnoreCase) + .ThenBy(p => p.Name, StringComparer.OrdinalIgnoreCase) + .ThenBy(p => p.Version ?? string.Empty, StringComparer.OrdinalIgnoreCase) + .ThenBy(p => p.Build ?? string.Empty, StringComparer.OrdinalIgnoreCase)) + { + var platform = product.Platform is null ? string.Empty : Escape(product.Platform); + var name = Escape(product.Name); + var version = product.Version is null ? string.Empty : Escape(product.Version); + var build = product.Build is null ? string.Empty : Escape(product.Build); + + builder.Append(" "); + builder.AppendLine(); + builder.AppendLine($" "); + builder.AppendLine($" "); + builder.AppendLine($" "); + builder.AppendLine(" "); + } + + builder.AppendLine(" "); + builder.AppendLine("
    {name}{version}{build}
    "); + } + + if (dto.References.Count > 0) + { + builder.AppendLine("
    "); + builder.AppendLine("

    References

    "); + foreach (var reference in dto.References + .OrderBy(r => r.Url, StringComparer.OrdinalIgnoreCase)) + { + var title = reference.Title ?? string.Empty; + builder.AppendLine($" {Escape(title)}"); + } + builder.AppendLine("
    "); + } + + builder.AppendLine("
    "); + builder.AppendLine(""); + builder.AppendLine(""); + return builder.ToString(); + } + + private static void WriteFixture(string name, string contents) + { + var root = ResolveFixtureRoot(); + Directory.CreateDirectory(root); + var normalized = NormalizeLineEndings(contents); + + var sourcePath = Path.Combine(root, name); + File.WriteAllText(sourcePath, normalized); + + var outputPath = Path.Combine(AppContext.BaseDirectory, "Source", "Vndr", "Apple", "Fixtures", name); + Directory.CreateDirectory(Path.GetDirectoryName(outputPath)!); + File.WriteAllText(outputPath, normalized); + + Console.WriteLine($"[AppleFixtures] Wrote {name}"); + } + + private static string ResolveFixtureRoot() + { + var baseDir = AppContext.BaseDirectory; + // bin/Debug/net10.0/ -> project -> src -> repo root + var root = Path.GetFullPath(Path.Combine(baseDir, "..", "..", "..", "..", "..")); + return Path.Combine(root, "src", "StellaOps.Feedser.Source.Vndr.Apple.Tests", "Apple", "Fixtures"); + } + + private static string NormalizeLineEndings(string value) + => value.Replace("\r\n", "\n", StringComparison.Ordinal); + + private static string Escape(string value) + => System.Net.WebUtility.HtmlEncode(value); +} + +internal sealed record AppleFixtureDefinition( + string ArticleId, + Uri DetailUri, + bool? ForceRapidSecurityResponse = null, + IReadOnlyList? Products = null) +{ + public string HtmlFixtureName => $"{ArticleId}.html"; + public string ExpectedFixtureName => $"{ArticleId}.expected.json"; +} + +internal sealed record AppleFixtureProduct( + string Platform, + string Name, + string Version, + string Build); diff --git a/src/StellaOps.Feedser.Source.Vndr.Apple.Tests/Apple/AppleLiveRegressionTests.cs b/src/StellaOps.Feedser.Source.Vndr.Apple.Tests/Apple/AppleLiveRegressionTests.cs new file mode 100644 index 00000000..53026658 --- /dev/null +++ b/src/StellaOps.Feedser.Source.Vndr.Apple.Tests/Apple/AppleLiveRegressionTests.cs @@ -0,0 +1,60 @@ +using System; +using System.Collections.Generic; +using System.Text.Encodings.Web; +using System.Text.Json; +using System.Text.Json.Serialization; +using System.Threading.Tasks; +using StellaOps.Feedser.Source.Vndr.Apple.Internal; +using StellaOps.Feedser.Source.Vndr.Apple.Tests.Apple; +using Xunit; + +namespace StellaOps.Feedser.Source.Vndr.Apple.Tests; + +public sealed class AppleLiveRegressionTests +{ + private static readonly JsonSerializerOptions SerializerOptions = new(JsonSerializerDefaults.Web) + { + WriteIndented = false, + DefaultIgnoreCondition = JsonIgnoreCondition.WhenWritingNull, + Encoder = JavaScriptEncoder.UnsafeRelaxedJsonEscaping, + }; + + public static IEnumerable FixtureCases + { + get + { + foreach (var definition in AppleFixtureManager.Fixtures) + { + yield return new object[] { definition.ArticleId }; + } + } + } + + [Theory] + [MemberData(nameof(FixtureCases))] + public async Task Parser_SanitizedFixture_MatchesExpectedDto(string articleId) + { + var updateFlag = Environment.GetEnvironmentVariable("UPDATE_APPLE_FIXTURES"); + if (!string.IsNullOrEmpty(updateFlag)) + { + Console.WriteLine($"[AppleFixtures] UPDATE_APPLE_FIXTURES={updateFlag}"); + } + await AppleFixtureManager.EnsureUpdatedAsync(); + var expected = AppleFixtureManager.ReadExpectedDto(articleId); + var html = AppleFixtureManager.ReadFixture($"{articleId}.html"); + + var entry = new AppleIndexEntry( + UpdateId: articleId, + ArticleId: articleId, + Title: expected.Title, + PostingDate: expected.Published, + DetailUri: new Uri($"https://support.apple.com/en-us/{articleId}"), + Products: Array.Empty(), + IsRapidSecurityResponse: expected.RapidSecurityResponse); + + var dto = AppleDetailParser.Parse(html, entry); + var actualJson = JsonSerializer.Serialize(dto, SerializerOptions); + var expectedJson = JsonSerializer.Serialize(expected, SerializerOptions); + Assert.Equal(expectedJson, actualJson); + } +} diff --git a/src/StellaOps.Feedser.Source.Vndr.Apple.Tests/Apple/Fixtures/106355.expected.json b/src/StellaOps.Feedser.Source.Vndr.Apple.Tests/Apple/Fixtures/106355.expected.json new file mode 100644 index 00000000..1b16ba2a --- /dev/null +++ b/src/StellaOps.Feedser.Source.Vndr.Apple.Tests/Apple/Fixtures/106355.expected.json @@ -0,0 +1,151 @@ +{ + "advisoryId": "106355", + "articleId": "106355", + "title": "About the security content of Rapid Security Responses for macOS Ventura 13.4.1", + "summary": "This document describes the content of Rapid Security Responses.", + "published": "2025-10-11T10:51:13.5532383+00:00", + "cveIds": [ + "CVE-2023-37450" + ], + "affected": [ + { + "platform": "macOS Ventura", + "name": "macOS Ventura", + "version": "", + "build": "22F400" + }, + { + "platform": "macOS Ventura", + "name": "macOS Ventura (Intel)", + "version": "", + "build": "22F400a" + } + ], + "references": [ + { + "url": "https://support.apple.com/", + "title": "Support", + "kind": "advisory" + }, + { + "url": "https://support.apple.com/103190", + "title": "Contact the vendor", + "kind": "advisory" + }, + { + "url": "https://support.apple.com/?cid=gn-ols-home-hp-tab", + "title": "Support", + "kind": "advisory" + }, + { + "url": "https://support.apple.com/en-us/106355/localeselector", + "title": "United States", + "kind": "advisory" + }, + { + "url": "https://support.apple.com/kb/HT201222", + "title": "Apple security releases", + "kind": "advisory" + }, + { + "url": "https://support.apple.com/kb/HT201224", + "title": "Rapid Security Responses", + "kind": "advisory" + }, + { + "url": "https://support.apple.com/kb/index?page=search&locale=en_US", + "title": "", + "kind": "advisory" + }, + { + "url": "https://www.apple.com/", + "title": "\uF8FF Apple", + "kind": "advisory" + }, + { + "url": "https://www.apple.com/", + "title": "Apple", + "kind": "advisory" + }, + { + "url": "https://www.apple.com/airpods/", + "title": "AirPods", + "kind": "advisory" + }, + { + "url": "https://www.apple.com/apple-vision-pro/", + "title": "Vision", + "kind": "advisory" + }, + { + "url": "https://www.apple.com/entertainment/", + "title": "Entertainment", + "kind": "advisory" + }, + { + "url": "https://www.apple.com/ipad/", + "title": "iPad", + "kind": "advisory" + }, + { + "url": "https://www.apple.com/iphone/", + "title": "iPhone", + "kind": "advisory" + }, + { + "url": "https://www.apple.com/legal/internet-services/terms/site.html", + "title": "Terms of Use", + "kind": "advisory" + }, + { + "url": "https://www.apple.com/legal/privacy/en-ww/", + "title": "Privacy Policy", + "kind": "advisory" + }, + { + "url": "https://www.apple.com/mac/", + "title": "Mac", + "kind": "advisory" + }, + { + "url": "https://www.apple.com/shop/goto/help/sales_refunds", + "title": "Sales and Refunds", + "kind": "advisory" + }, + { + "url": "https://www.apple.com/sitemap/", + "title": "Site Map", + "kind": "advisory" + }, + { + "url": "https://www.apple.com/tv-home/", + "title": "TV & Home", + "kind": "advisory" + }, + { + "url": "https://www.apple.com/us/shop/goto/bag", + "title": "", + "kind": "advisory" + }, + { + "url": "https://www.apple.com/us/shop/goto/buy_accessories", + "title": "Accessories", + "kind": "advisory" + }, + { + "url": "https://www.apple.com/us/shop/goto/store", + "title": "Store", + "kind": "advisory" + }, + { + "url": "https://www.apple.com/watch/", + "title": "Watch", + "kind": "advisory" + }, + { + "url": "https://www.cve.org/About/Overview", + "title": "CVE-ID" + } + ], + "rapidSecurityResponse": true +} diff --git a/src/StellaOps.Feedser.Source.Vndr.Apple.Tests/Apple/Fixtures/106355.html b/src/StellaOps.Feedser.Source.Vndr.Apple.Tests/Apple/Fixtures/106355.html new file mode 100644 index 00000000..014e3009 --- /dev/null +++ b/src/StellaOps.Feedser.Source.Vndr.Apple.Tests/Apple/Fixtures/106355.html @@ -0,0 +1,58 @@ + + + +
    +

    About the security content of Rapid Security Responses for macOS Ventura 13.4.1

    +

    This document describes the content of Rapid Security Responses.

    + +
    +

    Security Issues

    +
      +
    • CVE-2023-37450
      • +
    +
    + + + + + + + + + + + + + +
    macOS Ventura22F400
    macOS Ventura (Intel)22F400a
    +
    +

    References

    + Support + Contact the vendor + Support + United States + Apple security releases + Rapid Security Responses + +  Apple + Apple + AirPods + Vision + Entertainment + iPad + iPhone + Terms of Use + Privacy Policy + Mac + Sales and Refunds + Site Map + TV & Home + + Accessories + Store + Watch + CVE-ID +
    +
    + + diff --git a/src/StellaOps.Feedser.Source.Vndr.Apple.Tests/Apple/Fixtures/125326.expected.json b/src/StellaOps.Feedser.Source.Vndr.Apple.Tests/Apple/Fixtures/125326.expected.json new file mode 100644 index 00000000..39e6c01a --- /dev/null +++ b/src/StellaOps.Feedser.Source.Vndr.Apple.Tests/Apple/Fixtures/125326.expected.json @@ -0,0 +1,157 @@ +{ + "advisoryId": "125326", + "articleId": "125326", + "title": "About the security content of iOS 26.0.1 and iPadOS 26.0.1", + "summary": "This document describes the security content of iOS 26.0.1 and iPadOS 26.0.1.", + "published": "2025-10-11T10:51:12.6963611+00:00", + "cveIds": [ + "CVE-2025-43400" + ], + "affected": [ + { + "platform": "iOS", + "name": "iPhone 16 Pro", + "version": "26.0.1", + "build": "24A341" + }, + { + "platform": "iOS", + "name": "iPhone 16 Pro", + "version": "26.0.1 (a)", + "build": "24A341a" + }, + { + "platform": "iPadOS", + "name": "iPad Pro (M4)", + "version": "26", + "build": "24B120" + } + ], + "references": [ + { + "url": "https://support.apple.com/", + "title": "Support", + "kind": "advisory" + }, + { + "url": "https://support.apple.com/103190", + "title": "Contact the vendor", + "kind": "advisory" + }, + { + "url": "https://support.apple.com/?cid=gn-ols-home-hp-tab", + "title": "Support", + "kind": "advisory" + }, + { + "url": "https://support.apple.com/en-us/100100", + "title": "Apple security releases", + "kind": "advisory" + }, + { + "url": "https://support.apple.com/en-us/102549", + "title": "Apple Product Security", + "kind": "advisory" + }, + { + "url": "https://support.apple.com/en-us/125326/localeselector", + "title": "United States", + "kind": "advisory" + }, + { + "url": "https://support.apple.com/kb/index?page=search&locale=en_US", + "title": "", + "kind": "advisory" + }, + { + "url": "https://www.apple.com/", + "title": "\uF8FF Apple", + "kind": "advisory" + }, + { + "url": "https://www.apple.com/", + "title": "Apple", + "kind": "advisory" + }, + { + "url": "https://www.apple.com/airpods/", + "title": "AirPods", + "kind": "advisory" + }, + { + "url": "https://www.apple.com/apple-vision-pro/", + "title": "Vision", + "kind": "advisory" + }, + { + "url": "https://www.apple.com/entertainment/", + "title": "Entertainment", + "kind": "advisory" + }, + { + "url": "https://www.apple.com/ipad/", + "title": "iPad", + "kind": "advisory" + }, + { + "url": "https://www.apple.com/iphone/", + "title": "iPhone", + "kind": "advisory" + }, + { + "url": "https://www.apple.com/legal/internet-services/terms/site.html", + "title": "Terms of Use", + "kind": "advisory" + }, + { + "url": "https://www.apple.com/legal/privacy/en-ww/", + "title": "Privacy Policy", + "kind": "advisory" + }, + { + "url": "https://www.apple.com/mac/", + "title": "Mac", + "kind": "advisory" + }, + { + "url": "https://www.apple.com/shop/goto/help/sales_refunds", + "title": "Sales and Refunds", + "kind": "advisory" + }, + { + "url": "https://www.apple.com/sitemap/", + "title": "Site Map", + "kind": "advisory" + }, + { + "url": "https://www.apple.com/tv-home/", + "title": "TV & Home", + "kind": "advisory" + }, + { + "url": "https://www.apple.com/us/shop/goto/bag", + "title": "", + "kind": "advisory" + }, + { + "url": "https://www.apple.com/us/shop/goto/buy_accessories", + "title": "Accessories", + "kind": "advisory" + }, + { + "url": "https://www.apple.com/us/shop/goto/store", + "title": "Store", + "kind": "advisory" + }, + { + "url": "https://www.apple.com/watch/", + "title": "Watch", + "kind": "advisory" + }, + { + "url": "https://www.cve.org/About/Overview", + "title": "CVE-ID" + } + ], + "rapidSecurityResponse": false +} diff --git a/src/StellaOps.Feedser.Source.Vndr.Apple.Tests/Apple/Fixtures/125326.html b/src/StellaOps.Feedser.Source.Vndr.Apple.Tests/Apple/Fixtures/125326.html new file mode 100644 index 00000000..dbbdc98a --- /dev/null +++ b/src/StellaOps.Feedser.Source.Vndr.Apple.Tests/Apple/Fixtures/125326.html @@ -0,0 +1,63 @@ + + + +
    +

    About the security content of iOS 26.0.1 and iPadOS 26.0.1

    +

    This document describes the security content of iOS 26.0.1 and iPadOS 26.0.1.

    + +
    +

    Security Issues

    +
      +
    • CVE-2025-43400
      • +
    +
    + + + + + + + + + + + + + + + + + + +
    iPhone 16 Pro26.0.124A341
    iPhone 16 Pro26.0.1 (a)24A341a
    iPad Pro (M4)2624B120
    +
    +

    References

    + Support + Contact the vendor + Support + Apple security releases + Apple Product Security + United States + +  Apple + Apple + AirPods + Vision + Entertainment + iPad + iPhone + Terms of Use + Privacy Policy + Mac + Sales and Refunds + Site Map + TV & Home + + Accessories + Store + Watch + CVE-ID +
    +
    + + diff --git a/src/StellaOps.Feedser.Source.Vndr.Apple.Tests/Apple/Fixtures/125328.expected.json b/src/StellaOps.Feedser.Source.Vndr.Apple.Tests/Apple/Fixtures/125328.expected.json new file mode 100644 index 00000000..941ff76f --- /dev/null +++ b/src/StellaOps.Feedser.Source.Vndr.Apple.Tests/Apple/Fixtures/125328.expected.json @@ -0,0 +1,151 @@ +{ + "advisoryId": "125328", + "articleId": "125328", + "title": "About the security content of macOS Tahoe 26.0.1", + "summary": "This document describes the security content of macOS Tahoe 26.0.1.", + "published": "2025-10-11T10:51:13.4934166+00:00", + "cveIds": [ + "CVE-2025-43400" + ], + "affected": [ + { + "platform": "macOS", + "name": "Mac Studio", + "version": "26", + "build": "26A120b" + }, + { + "platform": "macOS", + "name": "MacBook Pro (M4)", + "version": "26.0.1", + "build": "26A123" + } + ], + "references": [ + { + "url": "https://support.apple.com/", + "title": "Support", + "kind": "advisory" + }, + { + "url": "https://support.apple.com/103190", + "title": "Contact the vendor", + "kind": "advisory" + }, + { + "url": "https://support.apple.com/?cid=gn-ols-home-hp-tab", + "title": "Support", + "kind": "advisory" + }, + { + "url": "https://support.apple.com/en-us/100100", + "title": "Apple security releases", + "kind": "advisory" + }, + { + "url": "https://support.apple.com/en-us/102549", + "title": "Apple Product Security", + "kind": "advisory" + }, + { + "url": "https://support.apple.com/en-us/125328/localeselector", + "title": "United States", + "kind": "advisory" + }, + { + "url": "https://support.apple.com/kb/index?page=search&locale=en_US", + "title": "", + "kind": "advisory" + }, + { + "url": "https://www.apple.com/", + "title": "\uF8FF Apple", + "kind": "advisory" + }, + { + "url": "https://www.apple.com/", + "title": "Apple", + "kind": "advisory" + }, + { + "url": "https://www.apple.com/airpods/", + "title": "AirPods", + "kind": "advisory" + }, + { + "url": "https://www.apple.com/apple-vision-pro/", + "title": "Vision", + "kind": "advisory" + }, + { + "url": "https://www.apple.com/entertainment/", + "title": "Entertainment", + "kind": "advisory" + }, + { + "url": "https://www.apple.com/ipad/", + "title": "iPad", + "kind": "advisory" + }, + { + "url": "https://www.apple.com/iphone/", + "title": "iPhone", + "kind": "advisory" + }, + { + "url": "https://www.apple.com/legal/internet-services/terms/site.html", + "title": "Terms of Use", + "kind": "advisory" + }, + { + "url": "https://www.apple.com/legal/privacy/en-ww/", + "title": "Privacy Policy", + "kind": "advisory" + }, + { + "url": "https://www.apple.com/mac/", + "title": "Mac", + "kind": "advisory" + }, + { + "url": "https://www.apple.com/shop/goto/help/sales_refunds", + "title": "Sales and Refunds", + "kind": "advisory" + }, + { + "url": "https://www.apple.com/sitemap/", + "title": "Site Map", + "kind": "advisory" + }, + { + "url": "https://www.apple.com/tv-home/", + "title": "TV & Home", + "kind": "advisory" + }, + { + "url": "https://www.apple.com/us/shop/goto/bag", + "title": "", + "kind": "advisory" + }, + { + "url": "https://www.apple.com/us/shop/goto/buy_accessories", + "title": "Accessories", + "kind": "advisory" + }, + { + "url": "https://www.apple.com/us/shop/goto/store", + "title": "Store", + "kind": "advisory" + }, + { + "url": "https://www.apple.com/watch/", + "title": "Watch", + "kind": "advisory" + }, + { + "url": "https://www.cve.org/About/Overview", + "title": "CVE-ID" + } + ], + "rapidSecurityResponse": false +} diff --git a/src/StellaOps.Feedser.Source.Vndr.Apple.Tests/Apple/Fixtures/125328.html b/src/StellaOps.Feedser.Source.Vndr.Apple.Tests/Apple/Fixtures/125328.html new file mode 100644 index 00000000..f83dcd13 --- /dev/null +++ b/src/StellaOps.Feedser.Source.Vndr.Apple.Tests/Apple/Fixtures/125328.html @@ -0,0 +1,58 @@ + + + +
    +

    About the security content of macOS Tahoe 26.0.1

    +

    This document describes the security content of macOS Tahoe 26.0.1.

    + +
    +

    Security Issues

    +
      +
    • CVE-2025-43400
      • +
    +
    + + + + + + + + + + + + + +
    Mac Studio2626A120b
    MacBook Pro (M4)26.0.126A123
    +
    +

    References

    + Support + Contact the vendor + Support + Apple security releases + Apple Product Security + United States + +  Apple + Apple + AirPods + Vision + Entertainment + iPad + iPhone + Terms of Use + Privacy Policy + Mac + Sales and Refunds + Site Map + TV & Home + + Accessories + Store + Watch + CVE-ID +
    +
    + + diff --git a/src/StellaOps.Feedser.Source.Vndr.Apple.Tests/Apple/Fixtures/ht214108.html b/src/StellaOps.Feedser.Source.Vndr.Apple.Tests/Apple/Fixtures/ht214108.html new file mode 100644 index 00000000..26627725 --- /dev/null +++ b/src/StellaOps.Feedser.Source.Vndr.Apple.Tests/Apple/Fixtures/ht214108.html @@ -0,0 +1,52 @@ + + + + + About the security content of iOS 18.0.1 + + + +
    +

    About the security content of iOS 18.0.1

    +

    + This update provides important security fixes for iPhone and is recommended for all users. +

    + + + +
    +

    Impact

    +
      +
    • An out-of-bounds write was addressed with improved bounds checking. (CVE-2025-1234)
      • +
    • A logic issue was addressed with improved state management. (CVE-2025-5678)
      • +
    +
    + + + + + + + + + + + + + + + + +
    ProductVersionBuild
    iPhone 15 Pro18.0.122A123
    + +

    + For more information, visit the + download page. +

    +
    + + diff --git a/src/StellaOps.Feedser.Source.Vndr.Apple.Tests/Apple/Fixtures/ht215500.html b/src/StellaOps.Feedser.Source.Vndr.Apple.Tests/Apple/Fixtures/ht215500.html new file mode 100644 index 00000000..024865d2 --- /dev/null +++ b/src/StellaOps.Feedser.Source.Vndr.Apple.Tests/Apple/Fixtures/ht215500.html @@ -0,0 +1,47 @@ + + + + + Rapid Security Response iOS 18.0.1 (c) + + +
    +

    Rapid Security Response iOS 18.0.1 (c)

    +

    + Rapid Security Response provides important security fixes between software updates. +

    + + +
    +

    + This update is recommended for all iPhone users and addresses CVE-2025-2468. +

    +
    + +
      +
    • CVE-2025-2468: WebKit
      • +
    + + + + + + + + + + + + + + + + +
    ProductVersionBuild
    iPhone 15 Pro18.0.1 (c)22A123c
    +
    + + diff --git a/src/StellaOps.Feedser.Source.Vndr.Apple.Tests/Apple/Fixtures/index.json b/src/StellaOps.Feedser.Source.Vndr.Apple.Tests/Apple/Fixtures/index.json new file mode 100644 index 00000000..b9408223 --- /dev/null +++ b/src/StellaOps.Feedser.Source.Vndr.Apple.Tests/Apple/Fixtures/index.json @@ -0,0 +1,36 @@ +{ + "updates": [ + { + "id": "HT214108", + "articleId": "HT214108", + "title": "About the security content of iOS 18.0.1", + "postingDate": "2025-09-29T10:00:00Z", + "detailUrl": "https://support.example.com/en-us/HT214108", + "rapidSecurityResponse": false, + "products": [ + { + "platform": "iOS", + "name": "iPhone 15 Pro", + "version": "18.0.1", + "build": "22A123" + } + ] + }, + { + "id": "RSR-iOS-18.0.1-c", + "articleId": "HT215500", + "title": "Rapid Security Response iOS 18.0.1 (c)", + "postingDate": "2025-10-02T15:30:00Z", + "detailUrl": "https://support.example.com/en-us/HT215500", + "rapidSecurityResponse": true, + "products": [ + { + "platform": "iOS", + "name": "iPhone 15 Pro", + "version": "18.0.1 (c)", + "build": "22A123c" + } + ] + } + ] +} diff --git a/src/StellaOps.Feedser.Source.Vndr.Apple.Tests/StellaOps.Feedser.Source.Vndr.Apple.Tests.csproj b/src/StellaOps.Feedser.Source.Vndr.Apple.Tests/StellaOps.Feedser.Source.Vndr.Apple.Tests.csproj new file mode 100644 index 00000000..dbd05cbe --- /dev/null +++ b/src/StellaOps.Feedser.Source.Vndr.Apple.Tests/StellaOps.Feedser.Source.Vndr.Apple.Tests.csproj @@ -0,0 +1,18 @@ + + + net10.0 + enable + enable + + + + + + + + + + + + + diff --git a/src/StellaOps.Feedser.Source.Vndr.Apple/AGENTS.md b/src/StellaOps.Feedser.Source.Vndr.Apple/AGENTS.md new file mode 100644 index 00000000..9bb992e0 --- /dev/null +++ b/src/StellaOps.Feedser.Source.Vndr.Apple/AGENTS.md @@ -0,0 +1,39 @@ +# AGENTS +## Role +Implement the Apple security advisories connector to ingest Apple HT/HT2 security bulletins for macOS/iOS/tvOS/visionOS. + +## Scope +- Identify canonical Apple security bulletin feeds (HTML, RSS, JSON) and change detection strategy. +- Implement fetch/cursor pipeline with retry/backoff, handling localisation/HTML quirks. +- Parse advisories to extract summary, affected products/versions, mitigation, CVEs. +- Map advisories into canonical `Advisory` records with aliases, references, affected packages, and range primitives (SemVer + vendor extensions). +- Produce deterministic fixtures and regression tests. + +## Participants +- `Source.Common` (HTTP/fetch utilities, DTO storage). +- `Storage.Mongo` (raw/document/DTO/advisory stores, source state). +- `Feedser.Models` (canonical structures + range primitives). +- `Feedser.Testing` (integration fixtures/snapshots). + +## Interfaces & Contracts +- Job kinds: `apple:fetch`, `apple:parse`, `apple:map`. +- Persist upstream metadata (ETag/Last-Modified or revision IDs) for incremental updates. +- Alias set should include Apple HT IDs and CVE IDs. + +## In/Out of scope +In scope: +- Security advisories covering Apple OS/app updates. +- Range primitives capturing device/OS version ranges. + +Out of scope: +- Release notes unrelated to security. + +## Observability & Security Expectations +- Log fetch/mapping statistics and failure details. +- Sanitize HTML while preserving structured data tables. +- Respect upstream rate limits; record failures with backoff. + +## Tests +- Add `StellaOps.Feedser.Source.Vndr.Apple.Tests` covering fetch/parse/map with fixtures. +- Snapshot canonical advisories; support fixture regeneration via env flag. +- Ensure deterministic ordering/time normalisation. diff --git a/src/StellaOps.Feedser.Source.Vndr.Apple/AppleConnector.cs b/src/StellaOps.Feedser.Source.Vndr.Apple/AppleConnector.cs new file mode 100644 index 00000000..01828324 --- /dev/null +++ b/src/StellaOps.Feedser.Source.Vndr.Apple/AppleConnector.cs @@ -0,0 +1,439 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Net; +using System.Text.Json; +using System.Text.Json.Serialization; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Options; +using MongoDB.Bson; +using StellaOps.Feedser.Source.Common; +using StellaOps.Feedser.Source.Common.Fetch; +using StellaOps.Feedser.Source.Vndr.Apple.Internal; +using StellaOps.Feedser.Storage.Mongo; +using StellaOps.Feedser.Storage.Mongo.Advisories; +using StellaOps.Feedser.Storage.Mongo.Documents; +using StellaOps.Feedser.Storage.Mongo.Dtos; +using StellaOps.Feedser.Storage.Mongo.PsirtFlags; +using StellaOps.Plugin; + +namespace StellaOps.Feedser.Source.Vndr.Apple; + +public sealed class AppleConnector : IFeedConnector +{ + private static readonly JsonSerializerOptions SerializerOptions = new(JsonSerializerDefaults.Web) + { + DefaultIgnoreCondition = JsonIgnoreCondition.WhenWritingNull, + PropertyNameCaseInsensitive = true, + }; + + private readonly SourceFetchService _fetchService; + private readonly RawDocumentStorage _rawDocumentStorage; + private readonly IDocumentStore _documentStore; + private readonly IDtoStore _dtoStore; + private readonly IAdvisoryStore _advisoryStore; + private readonly IPsirtFlagStore _psirtFlagStore; + private readonly ISourceStateRepository _stateRepository; + private readonly AppleOptions _options; + private readonly AppleDiagnostics _diagnostics; + private readonly TimeProvider _timeProvider; + private readonly ILogger _logger; + + public AppleConnector( + SourceFetchService fetchService, + RawDocumentStorage rawDocumentStorage, + IDocumentStore documentStore, + IDtoStore dtoStore, + IAdvisoryStore advisoryStore, + IPsirtFlagStore psirtFlagStore, + ISourceStateRepository stateRepository, + AppleDiagnostics diagnostics, + IOptions options, + TimeProvider? timeProvider, + ILogger logger) + { + _fetchService = fetchService ?? throw new ArgumentNullException(nameof(fetchService)); + _rawDocumentStorage = rawDocumentStorage ?? throw new ArgumentNullException(nameof(rawDocumentStorage)); + _documentStore = documentStore ?? throw new ArgumentNullException(nameof(documentStore)); + _dtoStore = dtoStore ?? throw new ArgumentNullException(nameof(dtoStore)); + _advisoryStore = advisoryStore ?? throw new ArgumentNullException(nameof(advisoryStore)); + _psirtFlagStore = psirtFlagStore ?? throw new ArgumentNullException(nameof(psirtFlagStore)); + _stateRepository = stateRepository ?? throw new ArgumentNullException(nameof(stateRepository)); + _diagnostics = diagnostics ?? throw new ArgumentNullException(nameof(diagnostics)); + _options = (options ?? throw new ArgumentNullException(nameof(options))).Value ?? throw new ArgumentNullException(nameof(options)); + _options.Validate(); + _timeProvider = timeProvider ?? TimeProvider.System; + _logger = logger ?? throw new ArgumentNullException(nameof(logger)); + } + + public string SourceName => VndrAppleConnectorPlugin.SourceName; + + public async Task FetchAsync(IServiceProvider services, CancellationToken cancellationToken) + { + ArgumentNullException.ThrowIfNull(services); + + var cursor = await GetCursorAsync(cancellationToken).ConfigureAwait(false); + var pendingDocuments = cursor.PendingDocuments.ToHashSet(); + var pendingMappings = cursor.PendingMappings.ToHashSet(); + var processedIds = cursor.ProcessedIds.ToHashSet(StringComparer.OrdinalIgnoreCase); + var maxPosted = cursor.LastPosted ?? DateTimeOffset.MinValue; + var baseline = cursor.LastPosted?.Add(-_options.ModifiedTolerance) ?? _timeProvider.GetUtcNow().Add(-_options.InitialBackfill); + + SourceFetchContentResult indexResult; + try + { + var request = new SourceFetchRequest(AppleOptions.HttpClientName, SourceName, _options.SoftwareLookupUri!) + { + AcceptHeaders = new[] { "application/json", "application/vnd.apple.security+json;q=0.9" }, + }; + + indexResult = await _fetchService.FetchContentAsync(request, cancellationToken).ConfigureAwait(false); + } + catch (Exception ex) + { + _diagnostics.FetchFailure(); + _logger.LogError(ex, "Apple software index fetch failed from {Uri}", _options.SoftwareLookupUri); + await _stateRepository.MarkFailureAsync(SourceName, _timeProvider.GetUtcNow(), TimeSpan.FromMinutes(10), ex.Message, cancellationToken).ConfigureAwait(false); + throw; + } + + if (!indexResult.IsSuccess || indexResult.Content is null) + { + if (indexResult.IsNotModified) + { + _diagnostics.FetchUnchanged(); + } + + await UpdateCursorAsync(cursor, cancellationToken).ConfigureAwait(false); + return; + } + + var indexEntries = AppleIndexParser.Parse(indexResult.Content, _options.AdvisoryBaseUri!); + if (indexEntries.Count == 0) + { + await UpdateCursorAsync(cursor, cancellationToken).ConfigureAwait(false); + return; + } + + var allowlist = _options.AdvisoryAllowlist; + var blocklist = _options.AdvisoryBlocklist; + + var ordered = indexEntries + .Where(entry => ShouldInclude(entry, allowlist, blocklist)) + .OrderBy(entry => entry.PostingDate) + .ThenBy(entry => entry.ArticleId, StringComparer.OrdinalIgnoreCase) + .ToArray(); + + foreach (var entry in ordered) + { + cancellationToken.ThrowIfCancellationRequested(); + + if (entry.PostingDate < baseline) + { + continue; + } + + if (cursor.LastPosted.HasValue + && entry.PostingDate <= cursor.LastPosted.Value + && processedIds.Contains(entry.UpdateId)) + { + continue; + } + + var metadata = BuildMetadata(entry); + var existing = await _documentStore.FindBySourceAndUriAsync(SourceName, entry.DetailUri.ToString(), cancellationToken).ConfigureAwait(false); + + SourceFetchResult result; + try + { + result = await _fetchService.FetchAsync( + new SourceFetchRequest(AppleOptions.HttpClientName, SourceName, entry.DetailUri) + { + Metadata = metadata, + ETag = existing?.Etag, + LastModified = existing?.LastModified, + AcceptHeaders = new[] + { + "text/html", + "application/xhtml+xml", + "text/plain;q=0.5" + }, + }, + cancellationToken).ConfigureAwait(false); + } + catch (Exception ex) + { + _diagnostics.FetchFailure(); + _logger.LogError(ex, "Apple advisory fetch failed for {Uri}", entry.DetailUri); + await _stateRepository.MarkFailureAsync(SourceName, _timeProvider.GetUtcNow(), TimeSpan.FromMinutes(5), ex.Message, cancellationToken).ConfigureAwait(false); + throw; + } + + if (result.StatusCode == HttpStatusCode.NotModified) + { + _diagnostics.FetchUnchanged(); + } + + if (!result.IsSuccess || result.Document is null) + { + continue; + } + + _diagnostics.FetchItem(); + + pendingDocuments.Add(result.Document.Id); + processedIds.Add(entry.UpdateId); + + if (entry.PostingDate > maxPosted) + { + maxPosted = entry.PostingDate; + } + } + + var updated = cursor + .WithPendingDocuments(pendingDocuments) + .WithPendingMappings(pendingMappings) + .WithLastPosted(maxPosted == DateTimeOffset.MinValue ? cursor.LastPosted ?? DateTimeOffset.MinValue : maxPosted, processedIds); + + await UpdateCursorAsync(updated, cancellationToken).ConfigureAwait(false); + } + + public async Task ParseAsync(IServiceProvider services, CancellationToken cancellationToken) + { + ArgumentNullException.ThrowIfNull(services); + + var cursor = await GetCursorAsync(cancellationToken).ConfigureAwait(false); + if (cursor.PendingDocuments.Count == 0) + { + return; + } + + var remainingDocuments = cursor.PendingDocuments.ToHashSet(); + var pendingMappings = cursor.PendingMappings.ToHashSet(); + + foreach (var documentId in cursor.PendingDocuments) + { + cancellationToken.ThrowIfCancellationRequested(); + + var document = await _documentStore.FindAsync(documentId, cancellationToken).ConfigureAwait(false); + if (document is null) + { + remainingDocuments.Remove(documentId); + pendingMappings.Remove(documentId); + continue; + } + + if (!document.GridFsId.HasValue) + { + _diagnostics.ParseFailure(); + _logger.LogWarning("Apple document {DocumentId} missing GridFS payload", document.Id); + await _documentStore.UpdateStatusAsync(document.Id, DocumentStatuses.Failed, cancellationToken).ConfigureAwait(false); + remainingDocuments.Remove(documentId); + pendingMappings.Remove(documentId); + continue; + } + + AppleDetailDto dto; + try + { + var content = await _rawDocumentStorage.DownloadAsync(document.GridFsId.Value, cancellationToken).ConfigureAwait(false); + var html = System.Text.Encoding.UTF8.GetString(content); + var entry = RehydrateIndexEntry(document); + dto = AppleDetailParser.Parse(html, entry); + } + catch (Exception ex) + { + _diagnostics.ParseFailure(); + _logger.LogError(ex, "Apple parse failed for document {DocumentId}", document.Id); + await _documentStore.UpdateStatusAsync(document.Id, DocumentStatuses.Failed, cancellationToken).ConfigureAwait(false); + remainingDocuments.Remove(documentId); + pendingMappings.Remove(documentId); + continue; + } + + var json = JsonSerializer.Serialize(dto, SerializerOptions); + var payload = BsonDocument.Parse(json); + var validatedAt = _timeProvider.GetUtcNow(); + + var existingDto = await _dtoStore.FindByDocumentIdAsync(document.Id, cancellationToken).ConfigureAwait(false); + var dtoRecord = existingDto is null + ? new DtoRecord(Guid.NewGuid(), document.Id, SourceName, "apple.security.update.v1", payload, validatedAt) + : existingDto with + { + Payload = payload, + SchemaVersion = "apple.security.update.v1", + ValidatedAt = validatedAt, + }; + + await _dtoStore.UpsertAsync(dtoRecord, cancellationToken).ConfigureAwait(false); + await _documentStore.UpdateStatusAsync(document.Id, DocumentStatuses.PendingMap, cancellationToken).ConfigureAwait(false); + + remainingDocuments.Remove(documentId); + pendingMappings.Add(document.Id); + } + + var updatedCursor = cursor + .WithPendingDocuments(remainingDocuments) + .WithPendingMappings(pendingMappings); + + await UpdateCursorAsync(updatedCursor, cancellationToken).ConfigureAwait(false); + } + + public async Task MapAsync(IServiceProvider services, CancellationToken cancellationToken) + { + ArgumentNullException.ThrowIfNull(services); + + var cursor = await GetCursorAsync(cancellationToken).ConfigureAwait(false); + if (cursor.PendingMappings.Count == 0) + { + return; + } + + var pendingMappings = cursor.PendingMappings.ToHashSet(); + + foreach (var documentId in cursor.PendingMappings) + { + cancellationToken.ThrowIfCancellationRequested(); + + var document = await _documentStore.FindAsync(documentId, cancellationToken).ConfigureAwait(false); + if (document is null) + { + pendingMappings.Remove(documentId); + continue; + } + + var dtoRecord = await _dtoStore.FindByDocumentIdAsync(document.Id, cancellationToken).ConfigureAwait(false); + if (dtoRecord is null) + { + pendingMappings.Remove(documentId); + continue; + } + + AppleDetailDto dto; + try + { + dto = JsonSerializer.Deserialize(dtoRecord.Payload.ToJson(), SerializerOptions) + ?? throw new InvalidOperationException("Unable to deserialize Apple DTO."); + } + catch (Exception ex) + { + _logger.LogError(ex, "Apple DTO deserialization failed for document {DocumentId}", document.Id); + pendingMappings.Remove(documentId); + await _documentStore.UpdateStatusAsync(document.Id, DocumentStatuses.Failed, cancellationToken).ConfigureAwait(false); + continue; + } + + var (advisory, flag) = AppleMapper.Map(dto, document, dtoRecord); + _diagnostics.MapAffectedCount(advisory.AffectedPackages.Length); + + await _advisoryStore.UpsertAsync(advisory, cancellationToken).ConfigureAwait(false); + await _documentStore.UpdateStatusAsync(document.Id, DocumentStatuses.Mapped, cancellationToken).ConfigureAwait(false); + + if (flag is not null) + { + await _psirtFlagStore.UpsertAsync(flag, cancellationToken).ConfigureAwait(false); + } + + pendingMappings.Remove(documentId); + } + + var updatedCursor = cursor.WithPendingMappings(pendingMappings); + await UpdateCursorAsync(updatedCursor, cancellationToken).ConfigureAwait(false); + } + + private AppleIndexEntry RehydrateIndexEntry(DocumentRecord document) + { + var metadata = document.Metadata ?? new Dictionary(StringComparer.Ordinal); + metadata.TryGetValue("apple.articleId", out var articleId); + metadata.TryGetValue("apple.updateId", out var updateId); + metadata.TryGetValue("apple.title", out var title); + metadata.TryGetValue("apple.postingDate", out var postingDateRaw); + metadata.TryGetValue("apple.detailUri", out var detailUriRaw); + metadata.TryGetValue("apple.rapidResponse", out var rapidRaw); + metadata.TryGetValue("apple.products", out var productsJson); + + if (!DateTimeOffset.TryParse(postingDateRaw, out var postingDate)) + { + postingDate = document.FetchedAt; + } + + var detailUri = !string.IsNullOrWhiteSpace(detailUriRaw) && Uri.TryCreate(detailUriRaw, UriKind.Absolute, out var parsedUri) + ? parsedUri + : new Uri(_options.AdvisoryBaseUri!, articleId ?? document.Uri); + + var rapid = string.Equals(rapidRaw, "true", StringComparison.OrdinalIgnoreCase); + var products = DeserializeProducts(productsJson); + + return new AppleIndexEntry( + UpdateId: string.IsNullOrWhiteSpace(updateId) ? articleId ?? document.Uri : updateId, + ArticleId: articleId ?? document.Uri, + Title: title ?? document.Metadata?["apple.originalTitle"] ?? "Apple Security Update", + PostingDate: postingDate.ToUniversalTime(), + DetailUri: detailUri, + Products: products, + IsRapidSecurityResponse: rapid); + } + + private static IReadOnlyList DeserializeProducts(string? json) + { + if (string.IsNullOrWhiteSpace(json)) + { + return Array.Empty(); + } + + try + { + var products = JsonSerializer.Deserialize>(json, SerializerOptions); + return products is { Count: > 0 } ? products : Array.Empty(); + } + catch (JsonException) + { + return Array.Empty(); + } + } + + private static Dictionary BuildMetadata(AppleIndexEntry entry) + { + var metadata = new Dictionary(StringComparer.Ordinal) + { + ["apple.articleId"] = entry.ArticleId, + ["apple.updateId"] = entry.UpdateId, + ["apple.title"] = entry.Title, + ["apple.postingDate"] = entry.PostingDate.ToString("O"), + ["apple.detailUri"] = entry.DetailUri.ToString(), + ["apple.rapidResponse"] = entry.IsRapidSecurityResponse ? "true" : "false", + ["apple.products"] = JsonSerializer.Serialize(entry.Products, SerializerOptions), + }; + + return metadata; + } + + private static bool ShouldInclude(AppleIndexEntry entry, IReadOnlyCollection allowlist, IReadOnlyCollection blocklist) + { + if (allowlist.Count > 0 && !allowlist.Contains(entry.ArticleId)) + { + return false; + } + + if (blocklist.Count > 0 && blocklist.Contains(entry.ArticleId)) + { + return false; + } + + return true; + } + + private async Task GetCursorAsync(CancellationToken cancellationToken) + { + var state = await _stateRepository.TryGetAsync(SourceName, cancellationToken).ConfigureAwait(false); + return state is null ? AppleCursor.Empty : AppleCursor.FromBson(state.Cursor); + } + + private async Task UpdateCursorAsync(AppleCursor cursor, CancellationToken cancellationToken) + { + var document = cursor.ToBson(); + await _stateRepository.UpdateCursorAsync(SourceName, document, _timeProvider.GetUtcNow(), cancellationToken).ConfigureAwait(false); + } +} diff --git a/src/StellaOps.Feedser.Source.Vndr.Apple/AppleDependencyInjectionRoutine.cs b/src/StellaOps.Feedser.Source.Vndr.Apple/AppleDependencyInjectionRoutine.cs new file mode 100644 index 00000000..35924cf6 --- /dev/null +++ b/src/StellaOps.Feedser.Source.Vndr.Apple/AppleDependencyInjectionRoutine.cs @@ -0,0 +1,53 @@ +using System; +using Microsoft.Extensions.Configuration; +using Microsoft.Extensions.DependencyInjection; +using StellaOps.DependencyInjection; +using StellaOps.Feedser.Core.Jobs; + +namespace StellaOps.Feedser.Source.Vndr.Apple; + +public sealed class AppleDependencyInjectionRoutine : IDependencyInjectionRoutine +{ + private const string ConfigurationSection = "feedser:sources:apple"; + + public IServiceCollection Register(IServiceCollection services, IConfiguration configuration) + { + ArgumentNullException.ThrowIfNull(services); + ArgumentNullException.ThrowIfNull(configuration); + + services.AddAppleConnector(options => + { + configuration.GetSection(ConfigurationSection).Bind(options); + options.Validate(); + }); + + services.AddTransient(); + services.AddTransient(); + services.AddTransient(); + + services.PostConfigure(options => + { + EnsureJob(options, AppleJobKinds.Fetch, typeof(AppleFetchJob)); + EnsureJob(options, AppleJobKinds.Parse, typeof(AppleParseJob)); + EnsureJob(options, AppleJobKinds.Map, typeof(AppleMapJob)); + }); + + return services; + } + + private static void EnsureJob(JobSchedulerOptions options, string kind, Type jobType) + { + if (options.Definitions.ContainsKey(kind)) + { + return; + } + + options.Definitions[kind] = new JobDefinition( + kind, + jobType, + options.DefaultTimeout, + options.DefaultLeaseDuration, + CronExpression: null, + Enabled: true); + } +} diff --git a/src/StellaOps.Feedser.Source.Vndr.Apple/AppleOptions.cs b/src/StellaOps.Feedser.Source.Vndr.Apple/AppleOptions.cs new file mode 100644 index 00000000..558188d0 --- /dev/null +++ b/src/StellaOps.Feedser.Source.Vndr.Apple/AppleOptions.cs @@ -0,0 +1,101 @@ +using System; +using System.Collections.Generic; +using System.ComponentModel.DataAnnotations; + +namespace StellaOps.Feedser.Source.Vndr.Apple; + +public sealed class AppleOptions : IValidatableObject +{ + public const string HttpClientName = "feedser-vndr-apple"; + + /// + /// Gets or sets the JSON endpoint that lists software metadata (defaults to Apple Software Lookup Service). + /// + public Uri? SoftwareLookupUri { get; set; } = new("https://gdmf.apple.com/v2/pmv"); + + /// + /// Gets or sets the base URI for HT advisory pages (locale neutral); trailing slash required. + /// + public Uri? AdvisoryBaseUri { get; set; } = new("https://support.apple.com/"); + + /// + /// Gets or sets the locale segment inserted between the base URI and HT identifier, e.g. "en-us". + /// + public string LocaleSegment { get; set; } = "en-us"; + + /// + /// Maximum advisories to fetch per run; defaults to 50. + /// + public int MaxAdvisoriesPerFetch { get; set; } = 50; + + /// + /// Sliding backfill window for initial sync (defaults to 90 days). + /// + public TimeSpan InitialBackfill { get; set; } = TimeSpan.FromDays(90); + + /// + /// Tolerance added to the modified timestamp comparisons during resume. + /// + public TimeSpan ModifiedTolerance { get; set; } = TimeSpan.FromHours(1); + + /// + /// Optional allowlist of HT identifiers to include; empty means include all. + /// + public HashSet AdvisoryAllowlist { get; } = new(StringComparer.OrdinalIgnoreCase); + + /// + /// Optional blocklist of HT identifiers to skip (e.g. non-security bulletins that share the feed). + /// + public HashSet AdvisoryBlocklist { get; } = new(StringComparer.OrdinalIgnoreCase); + + public IEnumerable Validate(ValidationContext validationContext) + { + if (SoftwareLookupUri is null) + { + yield return new ValidationResult("SoftwareLookupUri must be provided.", new[] { nameof(SoftwareLookupUri) }); + } + else if (!SoftwareLookupUri.IsAbsoluteUri) + { + yield return new ValidationResult("SoftwareLookupUri must be absolute.", new[] { nameof(SoftwareLookupUri) }); + } + + if (AdvisoryBaseUri is null) + { + yield return new ValidationResult("AdvisoryBaseUri must be provided.", new[] { nameof(AdvisoryBaseUri) }); + } + else if (!AdvisoryBaseUri.IsAbsoluteUri) + { + yield return new ValidationResult("AdvisoryBaseUri must be absolute.", new[] { nameof(AdvisoryBaseUri) }); + } + + if (string.IsNullOrWhiteSpace(LocaleSegment)) + { + yield return new ValidationResult("LocaleSegment must be specified.", new[] { nameof(LocaleSegment) }); + } + + if (MaxAdvisoriesPerFetch <= 0) + { + yield return new ValidationResult("MaxAdvisoriesPerFetch must be greater than zero.", new[] { nameof(MaxAdvisoriesPerFetch) }); + } + + if (InitialBackfill <= TimeSpan.Zero) + { + yield return new ValidationResult("InitialBackfill must be positive.", new[] { nameof(InitialBackfill) }); + } + + if (ModifiedTolerance < TimeSpan.Zero) + { + yield return new ValidationResult("ModifiedTolerance cannot be negative.", new[] { nameof(ModifiedTolerance) }); + } + } + + public void Validate() + { + var context = new ValidationContext(this); + var results = new List(); + if (!Validator.TryValidateObject(this, context, results, validateAllProperties: true)) + { + throw new ValidationException(string.Join("; ", results)); + } + } +} diff --git a/src/StellaOps.Feedser.Source.Vndr.Apple/AppleServiceCollectionExtensions.cs b/src/StellaOps.Feedser.Source.Vndr.Apple/AppleServiceCollectionExtensions.cs new file mode 100644 index 00000000..7807b3f5 --- /dev/null +++ b/src/StellaOps.Feedser.Source.Vndr.Apple/AppleServiceCollectionExtensions.cs @@ -0,0 +1,44 @@ +using System; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.DependencyInjection.Extensions; +using Microsoft.Extensions.Options; +using StellaOps.Feedser.Source.Common.Http; +using StellaOps.Feedser.Source.Vndr.Apple.Internal; + +namespace StellaOps.Feedser.Source.Vndr.Apple; + +public static class AppleServiceCollectionExtensions +{ + public static IServiceCollection AddAppleConnector(this IServiceCollection services, Action configure) + { + ArgumentNullException.ThrowIfNull(services); + ArgumentNullException.ThrowIfNull(configure); + + services.AddOptions() + .Configure(configure) + .PostConfigure(static opts => opts.Validate()) + .ValidateOnStart(); + + services.AddSourceHttpClient(AppleOptions.HttpClientName, static (sp, clientOptions) => + { + var options = sp.GetRequiredService>().Value; + clientOptions.Timeout = TimeSpan.FromSeconds(30); + clientOptions.UserAgent = "StellaOps.Feedser.Apple/1.0"; + clientOptions.AllowedHosts.Clear(); + if (options.SoftwareLookupUri is not null) + { + clientOptions.AllowedHosts.Add(options.SoftwareLookupUri.Host); + } + + if (options.AdvisoryBaseUri is not null) + { + clientOptions.AllowedHosts.Add(options.AdvisoryBaseUri.Host); + } + }); + + services.TryAddSingleton(_ => TimeProvider.System); + services.AddSingleton(); + services.AddTransient(); + return services; + } +} diff --git a/src/StellaOps.Feedser.Source.Vndr.Apple/Internal/AppleCursor.cs b/src/StellaOps.Feedser.Source.Vndr.Apple/Internal/AppleCursor.cs new file mode 100644 index 00000000..839b0a44 --- /dev/null +++ b/src/StellaOps.Feedser.Source.Vndr.Apple/Internal/AppleCursor.cs @@ -0,0 +1,114 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using MongoDB.Bson; + +namespace StellaOps.Feedser.Source.Vndr.Apple.Internal; + +internal sealed record AppleCursor( + DateTimeOffset? LastPosted, + IReadOnlyCollection ProcessedIds, + IReadOnlyCollection PendingDocuments, + IReadOnlyCollection PendingMappings) +{ + private static readonly IReadOnlyCollection EmptyGuidCollection = Array.Empty(); + private static readonly IReadOnlyCollection EmptyStringCollection = Array.Empty(); + + public static AppleCursor Empty { get; } = new(null, EmptyStringCollection, EmptyGuidCollection, EmptyGuidCollection); + + public BsonDocument ToBson() + { + var document = new BsonDocument + { + ["pendingDocuments"] = new BsonArray(PendingDocuments.Select(id => id.ToString())), + ["pendingMappings"] = new BsonArray(PendingMappings.Select(id => id.ToString())), + }; + + if (LastPosted.HasValue) + { + document["lastPosted"] = LastPosted.Value.UtcDateTime; + } + + if (ProcessedIds.Count > 0) + { + document["processedIds"] = new BsonArray(ProcessedIds); + } + + return document; + } + + public static AppleCursor FromBson(BsonDocument? document) + { + if (document is null || document.ElementCount == 0) + { + return Empty; + } + + var lastPosted = document.TryGetValue("lastPosted", out var lastPostedValue) + ? ParseDate(lastPostedValue) + : null; + + var processedIds = document.TryGetValue("processedIds", out var processedValue) && processedValue is BsonArray processedArray + ? processedArray.OfType() + .Where(static value => value.BsonType == BsonType.String) + .Select(static value => value.AsString.Trim()) + .Where(static value => value.Length > 0) + .Distinct(StringComparer.OrdinalIgnoreCase) + .ToArray() + : EmptyStringCollection; + + var pendingDocuments = ReadGuidArray(document, "pendingDocuments"); + var pendingMappings = ReadGuidArray(document, "pendingMappings"); + + return new AppleCursor(lastPosted, processedIds, pendingDocuments, pendingMappings); + } + + public AppleCursor WithLastPosted(DateTimeOffset timestamp, IEnumerable? processedIds = null) + { + var ids = processedIds is null + ? ProcessedIds + : processedIds.Where(static id => !string.IsNullOrWhiteSpace(id)) + .Select(static id => id.Trim()) + .Distinct(StringComparer.OrdinalIgnoreCase) + .ToArray(); + + return this with + { + LastPosted = timestamp.ToUniversalTime(), + ProcessedIds = ids, + }; + } + + public AppleCursor WithPendingDocuments(IEnumerable? ids) + => this with { PendingDocuments = ids?.Distinct().ToArray() ?? EmptyGuidCollection }; + + public AppleCursor WithPendingMappings(IEnumerable? ids) + => this with { PendingMappings = ids?.Distinct().ToArray() ?? EmptyGuidCollection }; + + private static IReadOnlyCollection ReadGuidArray(BsonDocument document, string key) + { + if (!document.TryGetValue(key, out var value) || value is not BsonArray array) + { + return EmptyGuidCollection; + } + + var results = new List(array.Count); + foreach (var element in array) + { + if (Guid.TryParse(element.ToString(), out var guid)) + { + results.Add(guid); + } + } + + return results; + } + + private static DateTimeOffset? ParseDate(BsonValue value) + => value.BsonType switch + { + BsonType.DateTime => DateTime.SpecifyKind(value.ToUniversalTime(), DateTimeKind.Utc), + BsonType.String when DateTimeOffset.TryParse(value.AsString, out var parsed) => parsed.ToUniversalTime(), + _ => null, + }; +} diff --git a/src/StellaOps.Feedser.Source.Vndr.Apple/Internal/AppleDetailDto.cs b/src/StellaOps.Feedser.Source.Vndr.Apple/Internal/AppleDetailDto.cs new file mode 100644 index 00000000..2076dccd --- /dev/null +++ b/src/StellaOps.Feedser.Source.Vndr.Apple/Internal/AppleDetailDto.cs @@ -0,0 +1,50 @@ +using System; +using System.Collections.Generic; +using System.Linq; + +namespace StellaOps.Feedser.Source.Vndr.Apple.Internal; + +internal sealed record AppleDetailDto( + string AdvisoryId, + string ArticleId, + string Title, + string Summary, + DateTimeOffset Published, + DateTimeOffset? Updated, + IReadOnlyList CveIds, + IReadOnlyList Affected, + IReadOnlyList References, + bool RapidSecurityResponse); + +internal sealed record AppleAffectedProductDto( + string Platform, + string Name, + string Version, + string Build); + +internal sealed record AppleReferenceDto( + string Url, + string? Title, + string? Kind); + +internal static class AppleDetailDtoExtensions +{ + public static AppleDetailDto WithAffectedFallback(this AppleDetailDto dto, IEnumerable products) + { + if (dto.Affected.Count > 0) + { + return dto; + } + + var fallback = products + .Where(static product => !string.IsNullOrWhiteSpace(product.Version) || !string.IsNullOrWhiteSpace(product.Build)) + .Select(static product => new AppleAffectedProductDto( + product.Platform, + product.Name, + product.Version, + product.Build)) + .ToArray(); + + return fallback.Length == 0 ? dto : dto with { Affected = fallback }; + } +} diff --git a/src/StellaOps.Feedser.Source.Vndr.Apple/Internal/AppleDetailParser.cs b/src/StellaOps.Feedser.Source.Vndr.Apple/Internal/AppleDetailParser.cs new file mode 100644 index 00000000..13cfcd02 --- /dev/null +++ b/src/StellaOps.Feedser.Source.Vndr.Apple/Internal/AppleDetailParser.cs @@ -0,0 +1,326 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text.RegularExpressions; +using AngleSharp.Dom; +using AngleSharp.Html.Dom; +using AngleSharp.Html.Parser; + +namespace StellaOps.Feedser.Source.Vndr.Apple.Internal; + +internal static class AppleDetailParser +{ + private static readonly HtmlParser Parser = new(); + private static readonly Regex CveRegex = new(@"CVE-\d{4}-\d{4,7}", RegexOptions.Compiled | RegexOptions.IgnoreCase); + + public static AppleDetailDto Parse(string html, AppleIndexEntry entry) + { + if (string.IsNullOrWhiteSpace(html)) + { + throw new ArgumentException("HTML content must not be empty.", nameof(html)); + } + + var document = Parser.ParseDocument(html); + var title = ResolveTitle(document, entry.Title); + var summary = ResolveSummary(document); + var (published, updated) = ResolveTimestamps(document, entry.PostingDate); + var cves = ExtractCves(document); + var affected = ExtractProducts(document); + var references = ExtractReferences(document, entry.DetailUri); + + var dto = new AppleDetailDto( + entry.ArticleId, + entry.ArticleId, + title, + summary, + published, + updated, + cves, + affected, + references, + entry.IsRapidSecurityResponse); + + return dto.WithAffectedFallback(entry.Products); + } + + private static string ResolveTitle(IHtmlDocument document, string fallback) + { + var title = document.QuerySelector("[data-testid='update-title']")?.TextContent + ?? document.QuerySelector("h1, h2")?.TextContent + ?? document.Title; + + title = title?.Trim(); + return string.IsNullOrEmpty(title) ? fallback : title; + } + + private static string ResolveSummary(IHtmlDocument document) + { + var summary = document.QuerySelector("[data-testid='update-summary']")?.TextContent + ?? document.QuerySelector("meta[name='description']")?.GetAttribute("content") + ?? document.QuerySelector("p")?.TextContent + ?? string.Empty; + + return CleanWhitespace(summary); + } + + private static (DateTimeOffset Published, DateTimeOffset? Updated) ResolveTimestamps(IHtmlDocument document, DateTimeOffset postingFallback) + { + DateTimeOffset published = postingFallback; + DateTimeOffset? updated = null; + + foreach (var time in document.QuerySelectorAll("time")) + { + var raw = time.GetAttribute("datetime") ?? time.TextContent; + if (string.IsNullOrWhiteSpace(raw)) + { + continue; + } + + if (!DateTimeOffset.TryParse(raw, out var parsed)) + { + continue; + } + + parsed = parsed.ToUniversalTime(); + + var itemProp = time.GetAttribute("itemprop") ?? string.Empty; + var dataTestId = time.GetAttribute("data-testid") ?? string.Empty; + + if (itemProp.Equals("datePublished", StringComparison.OrdinalIgnoreCase) + || dataTestId.Equals("published", StringComparison.OrdinalIgnoreCase)) + { + published = parsed; + } + else if (itemProp.Equals("dateModified", StringComparison.OrdinalIgnoreCase) + || dataTestId.Equals("updated", StringComparison.OrdinalIgnoreCase)) + { + updated = parsed; + } + else if (updated is null && parsed > published) + { + updated = parsed; + } + } + + return (published, updated); + } + + private static IReadOnlyList ExtractCves(IHtmlDocument document) + { + var set = new HashSet(StringComparer.OrdinalIgnoreCase); + foreach (var node in document.All) + { + if (node.NodeType != NodeType.Text && node.NodeType != NodeType.Element) + { + continue; + } + + var text = node.TextContent; + if (string.IsNullOrWhiteSpace(text)) + { + continue; + } + + foreach (Match match in CveRegex.Matches(text)) + { + if (match.Success) + { + set.Add(match.Value.ToUpperInvariant()); + } + } + } + + if (set.Count == 0) + { + return Array.Empty(); + } + + var list = set.ToList(); + list.Sort(StringComparer.OrdinalIgnoreCase); + return list; + } + + private static IReadOnlyList ExtractProducts(IHtmlDocument document) + { + var rows = new List(); + + foreach (var element in document.QuerySelectorAll("[data-testid='product-row']")) + { + var platform = element.GetAttribute("data-platform") ?? string.Empty; + var name = element.GetAttribute("data-product") ?? platform; + var version = element.GetAttribute("data-version") ?? string.Empty; + var build = element.GetAttribute("data-build") ?? string.Empty; + + if (string.IsNullOrWhiteSpace(name) && element is IHtmlTableRowElement tableRow) + { + var cells = tableRow.Cells.Select(static cell => CleanWhitespace(cell.TextContent)).ToArray(); + if (cells.Length >= 1) + { + name = cells[0]; + } + + if (cells.Length >= 2 && string.IsNullOrWhiteSpace(version)) + { + version = cells[1]; + } + + if (cells.Length >= 3 && string.IsNullOrWhiteSpace(build)) + { + build = cells[2]; + } + } + + if (string.IsNullOrWhiteSpace(name)) + { + continue; + } + + rows.Add(new AppleAffectedProductDto(platform, name, version, build)); + } + + if (rows.Count > 0) + { + return rows; + } + + // fallback for generic tables without data attributes + foreach (var table in document.QuerySelectorAll("table")) + { + var headers = table.QuerySelectorAll("th").Select(static th => CleanWhitespace(th.TextContent)).ToArray(); + if (headers.Length == 0) + { + continue; + } + + var nameIndex = Array.FindIndex(headers, static header => header.Contains("product", StringComparison.OrdinalIgnoreCase) + || header.Contains("device", StringComparison.OrdinalIgnoreCase)); + var versionIndex = Array.FindIndex(headers, static header => header.Contains("version", StringComparison.OrdinalIgnoreCase)); + var buildIndex = Array.FindIndex(headers, static header => header.Contains("build", StringComparison.OrdinalIgnoreCase) + || header.Contains("release", StringComparison.OrdinalIgnoreCase)); + + if (nameIndex == -1 && versionIndex == -1 && buildIndex == -1) + { + continue; + } + + foreach (var row in table.QuerySelectorAll("tr")) + { + var cells = row.QuerySelectorAll("td").Select(static cell => CleanWhitespace(cell.TextContent)).ToArray(); + if (cells.Length == 0) + { + continue; + } + + string name = nameIndex >= 0 && nameIndex < cells.Length ? cells[nameIndex] : cells[0]; + string version = versionIndex >= 0 && versionIndex < cells.Length ? cells[versionIndex] : string.Empty; + string build = buildIndex >= 0 && buildIndex < cells.Length ? cells[buildIndex] : string.Empty; + + if (string.IsNullOrWhiteSpace(name)) + { + continue; + } + + rows.Add(new AppleAffectedProductDto(string.Empty, name, version, build)); + } + + if (rows.Count > 0) + { + break; + } + } + + return rows.Count == 0 ? Array.Empty() : rows; + } + + private static IReadOnlyList ExtractReferences(IHtmlDocument document, Uri detailUri) + { + var anchors = document.QuerySelectorAll("a[href]") + .Select(anchor => (Href: anchor.GetAttribute("href"), Title: CleanWhitespace(anchor.TextContent))) + .Where(static tuple => !string.IsNullOrWhiteSpace(tuple.Href)) + .ToArray(); + + if (anchors.Length == 0) + { + return Array.Empty(); + } + + var references = new List(anchors.Length); + foreach (var (href, title) in anchors) + { + if (!Uri.TryCreate(detailUri, href, out var uri)) + { + continue; + } + + var kind = ResolveReferenceKind(uri); + references.Add(new AppleReferenceDto(uri.ToString(), title, kind)); + } + + references.Sort(static (left, right) => StringComparer.OrdinalIgnoreCase.Compare(left.Url, right.Url)); + return references; + } + + private static string? ResolveReferenceKind(Uri uri) + { + if (uri.Host.Contains("apple.com", StringComparison.OrdinalIgnoreCase)) + { + if (uri.AbsolutePath.Contains("download", StringComparison.OrdinalIgnoreCase)) + { + return "download"; + } + + if (uri.AbsolutePath.Contains(".pdf", StringComparison.OrdinalIgnoreCase)) + { + return "document"; + } + + return "advisory"; + } + + if (uri.Host.Contains("nvd.nist.gov", StringComparison.OrdinalIgnoreCase)) + { + return "nvd"; + } + + if (uri.Host.Contains("support", StringComparison.OrdinalIgnoreCase)) + { + return "kb"; + } + + return null; + } + + private static string CleanWhitespace(string? value) + { + if (string.IsNullOrWhiteSpace(value)) + { + return string.Empty; + } + + var span = value.AsSpan(); + var buffer = new char[span.Length]; + var index = 0; + var previousWhitespace = false; + + foreach (var ch in span) + { + if (char.IsWhiteSpace(ch)) + { + if (previousWhitespace) + { + continue; + } + + buffer[index++] = ' '; + previousWhitespace = true; + } + else + { + buffer[index++] = ch; + previousWhitespace = false; + } + } + + return new string(buffer, 0, index).Trim(); + } +} diff --git a/src/StellaOps.Feedser.Source.Vndr.Apple/Internal/AppleDiagnostics.cs b/src/StellaOps.Feedser.Source.Vndr.Apple/Internal/AppleDiagnostics.cs new file mode 100644 index 00000000..caa94508 --- /dev/null +++ b/src/StellaOps.Feedser.Source.Vndr.Apple/Internal/AppleDiagnostics.cs @@ -0,0 +1,62 @@ +using System; +using System.Diagnostics.Metrics; + +namespace StellaOps.Feedser.Source.Vndr.Apple.Internal; + +public sealed class AppleDiagnostics : IDisposable +{ + public const string MeterName = "StellaOps.Feedser.Source.Vndr.Apple"; + private const string MeterVersion = "1.0.0"; + + private readonly Meter _meter; + private readonly Counter _fetchItems; + private readonly Counter _fetchFailures; + private readonly Counter _fetchUnchanged; + private readonly Counter _parseFailures; + private readonly Histogram _mapAffected; + + public AppleDiagnostics() + { + _meter = new Meter(MeterName, MeterVersion); + _fetchItems = _meter.CreateCounter( + name: "apple.fetch.items", + unit: "documents", + description: "Number of Apple advisories fetched."); + _fetchFailures = _meter.CreateCounter( + name: "apple.fetch.failures", + unit: "operations", + description: "Number of Apple fetch failures."); + _fetchUnchanged = _meter.CreateCounter( + name: "apple.fetch.unchanged", + unit: "documents", + description: "Number of Apple advisories skipped due to 304 responses."); + _parseFailures = _meter.CreateCounter( + name: "apple.parse.failures", + unit: "documents", + description: "Number of Apple documents that failed to parse."); + _mapAffected = _meter.CreateHistogram( + name: "apple.map.affected.count", + unit: "packages", + description: "Distribution of affected package counts emitted per Apple advisory."); + } + + public Meter Meter => _meter; + + public void FetchItem() => _fetchItems.Add(1); + + public void FetchFailure() => _fetchFailures.Add(1); + + public void FetchUnchanged() => _fetchUnchanged.Add(1); + + public void ParseFailure() => _parseFailures.Add(1); + + public void MapAffectedCount(int count) + { + if (count >= 0) + { + _mapAffected.Record(count); + } + } + + public void Dispose() => _meter.Dispose(); +} diff --git a/src/StellaOps.Feedser.Source.Vndr.Apple/Internal/AppleIndexEntry.cs b/src/StellaOps.Feedser.Source.Vndr.Apple/Internal/AppleIndexEntry.cs new file mode 100644 index 00000000..387769f1 --- /dev/null +++ b/src/StellaOps.Feedser.Source.Vndr.Apple/Internal/AppleIndexEntry.cs @@ -0,0 +1,144 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text.Json; +using System.Text.Json.Serialization; + +namespace StellaOps.Feedser.Source.Vndr.Apple.Internal; + +internal sealed record AppleIndexEntry( + string UpdateId, + string ArticleId, + string Title, + DateTimeOffset PostingDate, + Uri DetailUri, + IReadOnlyList Products, + bool IsRapidSecurityResponse); + +internal sealed record AppleIndexProduct( + string Platform, + string Name, + string Version, + string Build); + +internal static class AppleIndexParser +{ + private sealed record AppleIndexDocument( + [property: JsonPropertyName("updates")] IReadOnlyList? Updates); + + private sealed record AppleIndexEntryDto( + [property: JsonPropertyName("id")] string? Id, + [property: JsonPropertyName("articleId")] string? ArticleId, + [property: JsonPropertyName("title")] string? Title, + [property: JsonPropertyName("postingDate")] string? PostingDate, + [property: JsonPropertyName("detailUrl")] string? DetailUrl, + [property: JsonPropertyName("rapidSecurityResponse")] bool? RapidSecurityResponse, + [property: JsonPropertyName("products")] IReadOnlyList? Products); + + private sealed record AppleIndexProductDto( + [property: JsonPropertyName("platform")] string? Platform, + [property: JsonPropertyName("name")] string? Name, + [property: JsonPropertyName("version")] string? Version, + [property: JsonPropertyName("build")] string? Build); + + public static IReadOnlyList Parse(ReadOnlySpan payload, Uri baseUri) + { + if (payload.IsEmpty) + { + return Array.Empty(); + } + + var options = new JsonSerializerOptions(JsonSerializerDefaults.Web) + { + PropertyNameCaseInsensitive = true, + }; + + AppleIndexDocument? document; + try + { + document = JsonSerializer.Deserialize(payload, options); + } + catch (JsonException) + { + return Array.Empty(); + } + + if (document?.Updates is null || document.Updates.Count == 0) + { + return Array.Empty(); + } + + var entries = new List(document.Updates.Count); + foreach (var dto in document.Updates) + { + if (dto is null) + { + continue; + } + + var id = string.IsNullOrWhiteSpace(dto.Id) ? dto.ArticleId : dto.Id; + if (string.IsNullOrWhiteSpace(id) || string.IsNullOrWhiteSpace(dto.ArticleId)) + { + continue; + } + + if (string.IsNullOrWhiteSpace(dto.Title) || string.IsNullOrWhiteSpace(dto.PostingDate)) + { + continue; + } + + if (!DateTimeOffset.TryParse(dto.PostingDate, out var postingDate)) + { + continue; + } + + if (!TryResolveDetailUri(dto, baseUri, out var detailUri)) + { + continue; + } + + var products = dto.Products?.Select(static product => new AppleIndexProduct( + product.Platform ?? string.Empty, + product.Name ?? product.Platform ?? string.Empty, + product.Version ?? string.Empty, + product.Build ?? string.Empty)) + .ToArray() ?? Array.Empty(); + + entries.Add(new AppleIndexEntry( + id.Trim(), + dto.ArticleId!.Trim(), + dto.Title!.Trim(), + postingDate.ToUniversalTime(), + detailUri, + products, + dto.RapidSecurityResponse ?? false)); + } + + return entries.Count == 0 ? Array.Empty() : entries; + } + + private static bool TryResolveDetailUri(AppleIndexEntryDto dto, Uri baseUri, out Uri uri) + { + if (!string.IsNullOrWhiteSpace(dto.DetailUrl) && Uri.TryCreate(dto.DetailUrl, UriKind.Absolute, out uri)) + { + return true; + } + + if (string.IsNullOrWhiteSpace(dto.ArticleId)) + { + uri = default!; + return false; + } + + var article = dto.ArticleId.Trim(); + if (article.Length == 0) + { + uri = default!; + return false; + } + + var combined = new Uri(baseUri, article); + uri = combined; + return true; + } +} diff --git a/src/StellaOps.Feedser.Source.Vndr.Apple/Internal/AppleMapper.cs b/src/StellaOps.Feedser.Source.Vndr.Apple/Internal/AppleMapper.cs new file mode 100644 index 00000000..3af85899 --- /dev/null +++ b/src/StellaOps.Feedser.Source.Vndr.Apple/Internal/AppleMapper.cs @@ -0,0 +1,244 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using StellaOps.Feedser.Models; +using StellaOps.Feedser.Source.Common; +using StellaOps.Feedser.Source.Common.Packages; +using StellaOps.Feedser.Storage.Mongo.Documents; +using StellaOps.Feedser.Storage.Mongo.Dtos; +using StellaOps.Feedser.Storage.Mongo.PsirtFlags; + +namespace StellaOps.Feedser.Source.Vndr.Apple.Internal; + +internal static class AppleMapper +{ + public static (Advisory Advisory, PsirtFlagRecord? Flag) Map( + AppleDetailDto dto, + DocumentRecord document, + DtoRecord dtoRecord) + { + ArgumentNullException.ThrowIfNull(dto); + ArgumentNullException.ThrowIfNull(document); + ArgumentNullException.ThrowIfNull(dtoRecord); + + var recordedAt = dtoRecord.ValidatedAt.ToUniversalTime(); + + var fetchProvenance = new AdvisoryProvenance( + VndrAppleConnectorPlugin.SourceName, + "document", + document.Uri, + document.FetchedAt.ToUniversalTime()); + + var mapProvenance = new AdvisoryProvenance( + VndrAppleConnectorPlugin.SourceName, + "map", + dto.AdvisoryId, + recordedAt); + + var aliases = BuildAliases(dto); + var references = BuildReferences(dto, recordedAt); + var affected = BuildAffected(dto, recordedAt); + + var advisory = new Advisory( + advisoryKey: dto.AdvisoryId, + title: dto.Title, + summary: dto.Summary, + language: "en", + published: dto.Published.ToUniversalTime(), + modified: dto.Updated?.ToUniversalTime(), + severity: null, + exploitKnown: false, + aliases: aliases, + references: references, + affectedPackages: affected, + cvssMetrics: Array.Empty(), + provenance: new[] { fetchProvenance, mapProvenance }); + + PsirtFlagRecord? flag = dto.RapidSecurityResponse + ? new PsirtFlagRecord(dto.AdvisoryId, "Apple", VndrAppleConnectorPlugin.SourceName, dto.ArticleId, recordedAt) + : null; + + return (advisory, flag); + } + + private static IReadOnlyList BuildAliases(AppleDetailDto dto) + { + var set = new HashSet(StringComparer.OrdinalIgnoreCase) + { + dto.AdvisoryId, + dto.ArticleId, + }; + + foreach (var cve in dto.CveIds) + { + if (!string.IsNullOrWhiteSpace(cve)) + { + set.Add(cve.Trim()); + } + } + + var aliases = set.ToList(); + aliases.Sort(StringComparer.OrdinalIgnoreCase); + return aliases; + } + + private static IReadOnlyList BuildReferences(AppleDetailDto dto, DateTimeOffset recordedAt) + { + if (dto.References.Count == 0) + { + return Array.Empty(); + } + + var list = new List(dto.References.Count); + foreach (var reference in dto.References) + { + if (string.IsNullOrWhiteSpace(reference.Url)) + { + continue; + } + + try + { + var provenance = new AdvisoryProvenance( + VndrAppleConnectorPlugin.SourceName, + "reference", + reference.Url, + recordedAt); + + list.Add(new AdvisoryReference( + url: reference.Url, + kind: reference.Kind, + sourceTag: null, + summary: reference.Title, + provenance: provenance)); + } + catch (ArgumentException) + { + // ignore invalid URLs + } + } + + if (list.Count == 0) + { + return Array.Empty(); + } + + list.Sort(static (left, right) => StringComparer.OrdinalIgnoreCase.Compare(left.Url, right.Url)); + return list; + } + + private static IReadOnlyList BuildAffected(AppleDetailDto dto, DateTimeOffset recordedAt) + { + if (dto.Affected.Count == 0) + { + return Array.Empty(); + } + + var packages = new List(dto.Affected.Count); + foreach (var product in dto.Affected) + { + if (string.IsNullOrWhiteSpace(product.Name)) + { + continue; + } + + var provenance = new[] + { + new AdvisoryProvenance( + VndrAppleConnectorPlugin.SourceName, + "affected", + product.Name, + recordedAt), + }; + + var ranges = BuildRanges(product, recordedAt); + + packages.Add(new AffectedPackage( + type: AffectedPackageTypes.Vendor, + identifier: product.Name, + platform: product.Platform, + versionRanges: ranges, + statuses: Array.Empty(), + provenance: provenance)); + } + + return packages.Count == 0 ? Array.Empty() : packages; + } + + private static IReadOnlyList BuildRanges(AppleAffectedProductDto product, DateTimeOffset recordedAt) + { + if (string.IsNullOrWhiteSpace(product.Version) && string.IsNullOrWhiteSpace(product.Build)) + { + return Array.Empty(); + } + + var provenance = new AdvisoryProvenance( + VndrAppleConnectorPlugin.SourceName, + "range", + product.Name, + recordedAt); + + var extensions = new Dictionary(StringComparer.Ordinal); + if (!string.IsNullOrWhiteSpace(product.Version)) + { + extensions["apple.version.raw"] = product.Version; + } + + if (!string.IsNullOrWhiteSpace(product.Build)) + { + extensions["apple.build"] = product.Build; + } + + if (!string.IsNullOrWhiteSpace(product.Platform)) + { + extensions["apple.platform"] = product.Platform; + } + + var primitives = extensions.Count == 0 + ? null + : new RangePrimitives( + SemVer: TryCreateSemVerPrimitive(product.Version), + Nevra: null, + Evr: null, + VendorExtensions: extensions); + + var sanitizedVersion = PackageCoordinateHelper.TryParseSemVer(product.Version, out _, out var normalizedVersion) + ? normalizedVersion + : product.Version; + + return new[] + { + new AffectedVersionRange( + rangeKind: "vendor", + introducedVersion: null, + fixedVersion: sanitizedVersion, + lastAffectedVersion: null, + rangeExpression: product.Version, + provenance: provenance, + primitives: primitives), + }; + } + + private static SemVerPrimitive? TryCreateSemVerPrimitive(string? version) + { + if (string.IsNullOrWhiteSpace(version)) + { + return null; + } + + if (!PackageCoordinateHelper.TryParseSemVer(version, out _, out var normalized)) + { + return null; + } + + // treat as fixed version, unknown introduced/last affected + return new SemVerPrimitive( + Introduced: null, + IntroducedInclusive: true, + Fixed: normalized, + FixedInclusive: true, + LastAffected: null, + LastAffectedInclusive: true, + ConstraintExpression: null); + } +} diff --git a/src/StellaOps.Feedser.Source.Vndr.Apple/Jobs.cs b/src/StellaOps.Feedser.Source.Vndr.Apple/Jobs.cs new file mode 100644 index 00000000..92381260 --- /dev/null +++ b/src/StellaOps.Feedser.Source.Vndr.Apple/Jobs.cs @@ -0,0 +1,46 @@ +using System; +using System.Threading; +using System.Threading.Tasks; +using StellaOps.Feedser.Core.Jobs; + +namespace StellaOps.Feedser.Source.Vndr.Apple; + +internal static class AppleJobKinds +{ + public const string Fetch = "source:vndr-apple:fetch"; + public const string Parse = "source:vndr-apple:parse"; + public const string Map = "source:vndr-apple:map"; +} + +internal sealed class AppleFetchJob : IJob +{ + private readonly AppleConnector _connector; + + public AppleFetchJob(AppleConnector connector) + => _connector = connector ?? throw new ArgumentNullException(nameof(connector)); + + public Task ExecuteAsync(JobExecutionContext context, CancellationToken cancellationToken) + => _connector.FetchAsync(context.Services, cancellationToken); +} + +internal sealed class AppleParseJob : IJob +{ + private readonly AppleConnector _connector; + + public AppleParseJob(AppleConnector connector) + => _connector = connector ?? throw new ArgumentNullException(nameof(connector)); + + public Task ExecuteAsync(JobExecutionContext context, CancellationToken cancellationToken) + => _connector.ParseAsync(context.Services, cancellationToken); +} + +internal sealed class AppleMapJob : IJob +{ + private readonly AppleConnector _connector; + + public AppleMapJob(AppleConnector connector) + => _connector = connector ?? throw new ArgumentNullException(nameof(connector)); + + public Task ExecuteAsync(JobExecutionContext context, CancellationToken cancellationToken) + => _connector.MapAsync(context.Services, cancellationToken); +} diff --git a/src/StellaOps.Feedser.Source.Vndr.Apple/Properties/AssemblyInfo.cs b/src/StellaOps.Feedser.Source.Vndr.Apple/Properties/AssemblyInfo.cs new file mode 100644 index 00000000..ff1c1ce0 --- /dev/null +++ b/src/StellaOps.Feedser.Source.Vndr.Apple/Properties/AssemblyInfo.cs @@ -0,0 +1,3 @@ +using System.Runtime.CompilerServices; + +[assembly: InternalsVisibleTo("StellaOps.Feedser.Source.Vndr.Apple.Tests")] diff --git a/src/StellaOps.Feedser.Source.Vndr.Apple/README.md b/src/StellaOps.Feedser.Source.Vndr.Apple/README.md new file mode 100644 index 00000000..f70a2b8e --- /dev/null +++ b/src/StellaOps.Feedser.Source.Vndr.Apple/README.md @@ -0,0 +1,40 @@ +# Apple Security Updates Connector + +## Feed contract + +The Apple Software Lookup Service (`https://gdmf.apple.com/v2/pmv`) publishes JSON payloads describing every public software release Apple has shipped. Each `AssetSet` entry exposes: + +- `ProductBuildVersion`, `ProductVersion`, and channel flags (e.g., `RapidSecurityResponse`) +- Timestamps for `PostingDate`, `ExpirationDate`, and `PreInstallDeadline` +- Associated product families/devices (Mac, iPhone, iPad, Apple TV, Apple Watch, VisionOS) +- Metadata for download packages, release notes, and signing assets + +The service supports delta polling by filtering on `PostingDate` and `ReleaseType`; responses are gzip-compressed and require a standard HTTPS client.citeturn3search8 + +Apple’s new security updates landing hub (`https://support.apple.com/100100`) consolidates bulletin detail pages (HT articles). Each update is linked via an `HT` identifier such as `HT214108` and lists: + +- CVE identifiers with Apple’s internal tracking IDs +- Product version/build applicability tables +- Mitigation guidance, acknowledgements, and update packaging notesciteturn1search6 + +Historical advisories redirect to per-platform pages (e.g., macOS, iOS, visionOS). The HTML structure uses `
    ` blocks with nested tables for affected products. CVE rows include disclosure dates and impact text that we can normalise into canonical `AffectedPackage` entries. + +## Change detection strategy + +1. Poll the Software Lookup Service for updates where `PostingDate` is within the sliding window (`lastModified - tolerance`). Cache `ProductID` + `PostingDate` to avoid duplicate fetches. +2. For each candidate, derive the HT article URL from `DocumentationURL` or by combining the `HT` identifier with the base path (`https://support.apple.com/{locale}/`). Fetch with conditional headers (`If-None-Match`, `If-Modified-Since`). +3. On HTTP `200`, store the raw HTML + metadata (HT id, posting date, product identifiers). On `304`, re-queue existing documents for mapping only. + +Unofficial Apple documentation warns that the Software Lookup Service rate-limits clients after repeated unauthenticated bursts; respect 5 requests/second and honour `Retry-After` headers on `403/429` responses.citeturn3search3 + +## Parsing & mapping notes + +- CVE lists live inside `
      ` items; each `
    • ` contains CVE, impact, and credit text. Parse these into canonical `Alias` + `AffectedPackage` records, using Apple’s component name as the package `name` and the OS build as the range primitive seed. +- Product/version tables have headers for platform (`Platform`, `Version`, `Build`). Map the OS name into our vendor range primitive namespace (`apple.platform`, `apple.build`). +- Rapid Security Response advisories include an `Rapid Security Responses` badge; emit `psirt_flags` with `apple.rapid_security_response = true`. + +## Outstanding questions + +- Some HT pages embed downloadable PDFs for supplemental mitigations. Confirm whether to persist PDF text via the shared `PdfTextExtractor`. +- Vision Pro updates include `deviceFamily` identifiers not yet mapped in `RangePrimitives`. Extend the model with `apple.deviceFamily` once sample fixtures are captured. + diff --git a/src/StellaOps.Feedser.Source.Vndr.Apple/StellaOps.Feedser.Source.Vndr.Apple.csproj b/src/StellaOps.Feedser.Source.Vndr.Apple/StellaOps.Feedser.Source.Vndr.Apple.csproj new file mode 100644 index 00000000..c8aaf11c --- /dev/null +++ b/src/StellaOps.Feedser.Source.Vndr.Apple/StellaOps.Feedser.Source.Vndr.Apple.csproj @@ -0,0 +1,18 @@ + + + + net10.0 + enable + enable + + + + + + + + + + + + diff --git a/src/StellaOps.Feedser.Source.Vndr.Apple/TASKS.md b/src/StellaOps.Feedser.Source.Vndr.Apple/TASKS.md new file mode 100644 index 00000000..96a2860d --- /dev/null +++ b/src/StellaOps.Feedser.Source.Vndr.Apple/TASKS.md @@ -0,0 +1,11 @@ +# TASKS +| Task | Owner(s) | Depends on | Notes | +|---|---|---|---| +|Catalogue Apple security bulletin sources|BE-Conn-Apple|Research|**DONE** – Feed contract documented in README (Software Lookup Service JSON + HT article hub) with rate-limit notes.| +|Fetch pipeline & state persistence|BE-Conn-Apple|Source.Common, Storage.Mongo|**DONE** – Index fetch + detail ingestion with SourceState cursoring/allowlists committed; awaiting live smoke run before enabling in scheduler defaults.| +|Parser & DTO implementation|BE-Conn-Apple|Source.Common|**DONE** – AngleSharp detail parser produces canonical DTO payloads (CVE list, timestamps, affected tables) persisted via DTO store.| +|Canonical mapping & range primitives|BE-Conn-Apple|Models|**DOING** – Mapper emits aliases/references and vendor range primitives (apple.version/build/platform); extend coverage for additional device families and multi-range advisories.| +|Deterministic fixtures/tests|QA|Testing|**DOING** – Added canned index/detail fixtures and end-to-end connector test; TODO: wire `UPDATE_APPLE_FIXTURES=1` refresh workflow.| +|Telemetry & documentation|DevEx|Docs|**DOING** – Diagnostics meter + README shipped; need to integrate meters into global dashboard wiring.| +|Live HTML regression sweep|QA|Source.Common|**TODO** – Pull real support.apple.com snapshots across locales/builds, verify parser selectors, and baseline sanitized fixtures for macOS/iOS/watchOS variants (document gaps).| +|Fixture regeneration tooling|DevEx|Testing|**TODO** – Provide scripted `UPDATE_APPLE_FIXTURES=1` flow (docs + helper task) ensuring canonical JSON/HTML snapshots regenerate deterministically.| diff --git a/src/StellaOps.Feedser.Source.Vndr.Apple/VndrAppleConnectorPlugin.cs b/src/StellaOps.Feedser.Source.Vndr.Apple/VndrAppleConnectorPlugin.cs new file mode 100644 index 00000000..afa445f1 --- /dev/null +++ b/src/StellaOps.Feedser.Source.Vndr.Apple/VndrAppleConnectorPlugin.cs @@ -0,0 +1,24 @@ +using System; +using Microsoft.Extensions.DependencyInjection; +using StellaOps.Plugin; + +namespace StellaOps.Feedser.Source.Vndr.Apple; + +public sealed class VndrAppleConnectorPlugin : IConnectorPlugin +{ + public const string SourceName = "vndr-apple"; + + public string Name => SourceName; + + public bool IsAvailable(IServiceProvider services) + { + ArgumentNullException.ThrowIfNull(services); + return services.GetService() is not null; + } + + public IFeedConnector Create(IServiceProvider services) + { + ArgumentNullException.ThrowIfNull(services); + return services.GetRequiredService(); + } +} diff --git a/src/StellaOps.Feedser.Source.Vndr.Chromium.Tests/Chromium/ChromiumConnectorTests.cs b/src/StellaOps.Feedser.Source.Vndr.Chromium.Tests/Chromium/ChromiumConnectorTests.cs new file mode 100644 index 00000000..89fc0504 --- /dev/null +++ b/src/StellaOps.Feedser.Source.Vndr.Chromium.Tests/Chromium/ChromiumConnectorTests.cs @@ -0,0 +1,360 @@ +using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Http; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Logging.Abstractions; +using Microsoft.Extensions.Options; +using Microsoft.Extensions.Time.Testing; +using MongoDB.Bson; +using MongoDB.Driver; +using StellaOps.Feedser.Models; +using StellaOps.Feedser.Source.Common.Http; +using StellaOps.Feedser.Source.Common.Json; +using StellaOps.Feedser.Source.Common.Testing; +using StellaOps.Feedser.Source.Common; +using StellaOps.Feedser.Source.Vndr.Chromium; +using StellaOps.Feedser.Source.Vndr.Chromium.Configuration; +using StellaOps.Feedser.Storage.Mongo; +using StellaOps.Feedser.Storage.Mongo.Advisories; +using StellaOps.Feedser.Storage.Mongo.Documents; +using StellaOps.Feedser.Storage.Mongo.PsirtFlags; +using StellaOps.Feedser.Testing; + +namespace StellaOps.Feedser.Source.Vndr.Chromium.Tests; + +[Collection("mongo-fixture")] +public sealed class ChromiumConnectorTests : IAsyncLifetime +{ + private readonly MongoIntegrationFixture _fixture; + private readonly FakeTimeProvider _timeProvider; + private readonly List _allocatedDatabases = new(); + + public ChromiumConnectorTests(MongoIntegrationFixture fixture) + { + _fixture = fixture; + _timeProvider = new FakeTimeProvider(new DateTimeOffset(2024, 9, 10, 18, 0, 0, TimeSpan.Zero)); + } + + [Fact] + public async Task FetchParseMap_ProducesSnapshot() + { + var databaseName = AllocateDatabaseName(); + await DropDatabaseAsync(databaseName); + + try + { + var handler = new CannedHttpMessageHandler(); + await using var provider = await BuildServiceProviderAsync(handler, databaseName); + SeedHttpFixtures(handler); + + var connector = provider.GetRequiredService(); + await connector.FetchAsync(provider, CancellationToken.None); + try + { + await connector.ParseAsync(provider, CancellationToken.None); + } + catch (StellaOps.Feedser.Source.Common.Json.JsonSchemaValidationException) + { + // Parsing should flag document as failed even when schema validation rejects payloads. + } + await connector.MapAsync(provider, CancellationToken.None); + + var advisoryStore = provider.GetRequiredService(); + var advisories = await advisoryStore.GetRecentAsync(10, CancellationToken.None); + var advisory = Assert.Single(advisories); + + Assert.Equal("chromium/post/stable-channel-update-for-desktop", advisory.AdvisoryKey); + Assert.Contains("CHROMIUM-POST:stable-channel-update-for-desktop", advisory.Aliases); + Assert.Contains("CVE-2024-12345", advisory.Aliases); + Assert.Contains("CVE-2024-22222", advisory.Aliases); + + Assert.Contains(advisory.AffectedPackages, package => package.Platform == "android" && package.VersionRanges.Any(range => range.FixedVersion == "128.0.6613.89")); + Assert.Contains(advisory.AffectedPackages, package => package.Platform == "linux" && package.VersionRanges.Any(range => range.FixedVersion == "128.0.6613.137")); + Assert.Contains(advisory.AffectedPackages, package => package.Identifier == "google:chrome" && package.Platform == "windows-mac" && package.VersionRanges.Any(range => range.FixedVersion == "128.0.6613.138")); + Assert.Contains(advisory.AffectedPackages, package => package.Identifier == "google:chrome:extended-stable" && package.Platform == "windows-mac" && package.VersionRanges.Any(range => range.FixedVersion == "128.0.6613.138")); + + Assert.Contains(advisory.References, reference => reference.Url.Contains("chromium.googlesource.com", StringComparison.OrdinalIgnoreCase)); + Assert.Contains(advisory.References, reference => reference.Url.Contains("issues.chromium.org", StringComparison.OrdinalIgnoreCase)); + + var psirtStore = provider.GetRequiredService(); + var psirtFlag = await psirtStore.FindAsync(advisory.AdvisoryKey, CancellationToken.None); + Assert.NotNull(psirtFlag); + Assert.Equal("Google", psirtFlag!.Vendor); + + var canonicalJson = CanonicalJsonSerializer.Serialize(advisory).Trim(); + var snapshotPath = ResolveFixturePath("chromium-advisory.snapshot.json"); + var expected = File.ReadAllText(snapshotPath).Trim(); + if (!string.Equals(expected, canonicalJson, StringComparison.Ordinal)) + { + var actualPath = ResolveFixturePath("chromium-advisory.actual.json"); + Directory.CreateDirectory(Path.GetDirectoryName(actualPath)!); + File.WriteAllText(actualPath, canonicalJson); + } + Assert.Equal(expected, canonicalJson); + } + finally + { + await DropDatabaseAsync(databaseName); + } + } + + [Fact] + public async Task ParseFailure_MarksDocumentFailed() + { + var databaseName = AllocateDatabaseName(); + await DropDatabaseAsync(databaseName); + + try + { + var handler = new CannedHttpMessageHandler(); + var feedUri = new Uri("https://chromereleases.googleblog.com/atom.xml?max-results=50&start-index=1&redirect=false"); + var detailUri = new Uri("https://chromereleases.googleblog.com/2024/09/stable-channel-update-for-desktop.html"); + + handler.AddTextResponse(feedUri, ReadFixture("chromium-feed.xml"), "application/atom+xml"); + handler.AddTextResponse(detailUri, "
      missing post body
      ", "text/html"); + + await using var provider = await BuildServiceProviderAsync(handler, databaseName); + var connector = provider.GetRequiredService(); + + await connector.FetchAsync(provider, CancellationToken.None); + try + { + await connector.ParseAsync(provider, CancellationToken.None); + } + catch (JsonSchemaValidationException) + { + // Expected for malformed posts; connector should still flag the document as failed. + } + + var documentStore = provider.GetRequiredService(); + var document = await documentStore.FindBySourceAndUriAsync(VndrChromiumConnectorPlugin.SourceName, detailUri.ToString(), CancellationToken.None); + Assert.NotNull(document); + Assert.Equal(DocumentStatuses.Failed, document!.Status); + + var stateRepository = provider.GetRequiredService(); + var state = await stateRepository.TryGetAsync(VndrChromiumConnectorPlugin.SourceName, CancellationToken.None); + Assert.NotNull(state); + var pendingDocuments = state!.Cursor.TryGetValue("pendingDocuments", out var pendingDocsValue) + ? pendingDocsValue.AsBsonArray + : new BsonArray(); + Assert.Empty(pendingDocuments); + } + finally + { + await DropDatabaseAsync(databaseName); + } + } + + [Fact] + public async Task Resume_CompletesPendingDocumentsAfterRestart() + { + var databaseName = AllocateDatabaseName(); + await DropDatabaseAsync(databaseName); + + try + { + var fetchHandler = new CannedHttpMessageHandler(); + Guid[] pendingDocumentIds; + await using (var fetchProvider = await BuildServiceProviderAsync(fetchHandler, databaseName)) + { + SeedHttpFixtures(fetchHandler); + var connector = fetchProvider.GetRequiredService(); + await connector.FetchAsync(fetchProvider, CancellationToken.None); + + var stateRepository = fetchProvider.GetRequiredService(); + var state = await stateRepository.TryGetAsync(VndrChromiumConnectorPlugin.SourceName, CancellationToken.None); + Assert.NotNull(state); + var pendingDocuments = state!.Cursor.TryGetValue("pendingDocuments", out var pendingDocsValue) + ? pendingDocsValue.AsBsonArray + : new BsonArray(); + Assert.NotEmpty(pendingDocuments); + pendingDocumentIds = pendingDocuments.Select(value => Guid.Parse(value.AsString)).ToArray(); + } + + var resumeHandler = new CannedHttpMessageHandler(); + SeedHttpFixtures(resumeHandler); + await using var resumeProvider = await BuildServiceProviderAsync(resumeHandler, databaseName); + var stateRepositoryBefore = resumeProvider.GetRequiredService(); + var resumeState = await stateRepositoryBefore.TryGetAsync(VndrChromiumConnectorPlugin.SourceName, CancellationToken.None); + Assert.NotNull(resumeState); + var resumePendingDocs = resumeState!.Cursor.TryGetValue("pendingDocuments", out var resumePendingValue) + ? resumePendingValue.AsBsonArray + : new BsonArray(); + Assert.Equal(pendingDocumentIds.Length, resumePendingDocs.Count); + var resumeIds = resumePendingDocs.Select(value => Guid.Parse(value.AsString)).OrderBy(id => id).ToArray(); + Assert.Equal(pendingDocumentIds.OrderBy(id => id).ToArray(), resumeIds); + + var resumeConnector = resumeProvider.GetRequiredService(); + await resumeConnector.ParseAsync(resumeProvider, CancellationToken.None); + await resumeConnector.MapAsync(resumeProvider, CancellationToken.None); + + var documentStore = resumeProvider.GetRequiredService(); + foreach (var documentId in pendingDocumentIds) + { + var document = await documentStore.FindAsync(documentId, CancellationToken.None); + Assert.NotNull(document); + Assert.Equal(DocumentStatuses.Mapped, document!.Status); + } + + var stateRepositoryAfter = resumeProvider.GetRequiredService(); + var finalState = await stateRepositoryAfter.TryGetAsync(VndrChromiumConnectorPlugin.SourceName, CancellationToken.None); + Assert.NotNull(finalState); + var finalPending = finalState!.Cursor.TryGetValue("pendingDocuments", out var finalPendingDocs) + ? finalPendingDocs.AsBsonArray + : new BsonArray(); + Assert.Empty(finalPending); + + var finalPendingMappings = finalState.Cursor.TryGetValue("pendingMappings", out var finalPendingMappingsValue) + ? finalPendingMappingsValue.AsBsonArray + : new BsonArray(); + Assert.Empty(finalPendingMappings); + } + finally + { + await DropDatabaseAsync(databaseName); + } + } + + [Fact] + public async Task Fetch_SkipsUnchangedDocuments() + { + var databaseName = AllocateDatabaseName(); + await DropDatabaseAsync(databaseName); + + try + { + var handler = new CannedHttpMessageHandler(); + await using var provider = await BuildServiceProviderAsync(handler, databaseName); + SeedHttpFixtures(handler); + + var connector = provider.GetRequiredService(); + await connector.FetchAsync(provider, CancellationToken.None); + await connector.ParseAsync(provider, CancellationToken.None); + await connector.MapAsync(provider, CancellationToken.None); + + var advisoryStore = provider.GetRequiredService(); + var advisories = await advisoryStore.GetRecentAsync(10, CancellationToken.None); + Assert.Single(advisories); + + // Re-seed responses and fetch again with unchanged content. + SeedHttpFixtures(handler); + await connector.FetchAsync(provider, CancellationToken.None); + + var stateRepository = provider.GetRequiredService(); + var state = await stateRepository.TryGetAsync(VndrChromiumConnectorPlugin.SourceName, CancellationToken.None); + Assert.NotNull(state); + var cursor = state!.Cursor; + var pendingDocuments = cursor.TryGetValue("pendingDocuments", out var pendingDocsValue) + ? pendingDocsValue.AsBsonArray + : new BsonArray(); + Assert.Empty(pendingDocuments); + + var pendingMappings = cursor.TryGetValue("pendingMappings", out var pendingMappingsValue) + ? pendingMappingsValue.AsBsonArray + : new BsonArray(); + Assert.Empty(pendingMappings); + } + finally + { + await DropDatabaseAsync(databaseName); + } + } + + private async Task BuildServiceProviderAsync(CannedHttpMessageHandler handler, string databaseName) + { + var services = new ServiceCollection(); + services.AddLogging(builder => builder.AddProvider(NullLoggerProvider.Instance)); + services.AddSingleton(_timeProvider); + services.AddSingleton(handler); + + services.AddMongoStorage(options => + { + options.ConnectionString = _fixture.Runner.ConnectionString; + options.DatabaseName = databaseName; + options.CommandTimeout = TimeSpan.FromSeconds(5); + }); + + services.AddSourceCommon(); + services.AddChromiumConnector(opts => + { + opts.FeedUri = new Uri("https://chromereleases.googleblog.com/atom.xml"); + opts.InitialBackfill = TimeSpan.FromDays(30); + opts.WindowOverlap = TimeSpan.FromDays(1); + opts.MaxFeedPages = 1; + opts.MaxEntriesPerPage = 50; + }); + + services.Configure(ChromiumOptions.HttpClientName, builderOptions => + { + builderOptions.HttpMessageHandlerBuilderActions.Add(builder => + { + builder.PrimaryHandler = handler; + }); + }); + + var provider = services.BuildServiceProvider(); + + var bootstrapper = provider.GetRequiredService(); + await bootstrapper.InitializeAsync(CancellationToken.None); + + return provider; + } + + private string AllocateDatabaseName() + { + var name = $"chromium-tests-{Guid.NewGuid():N}"; + _allocatedDatabases.Add(name); + return name; + } + + private async Task DropDatabaseAsync(string databaseName) + { + try + { + await _fixture.Client.DropDatabaseAsync(databaseName); + } + catch (MongoCommandException ex) when (ex.CodeName == "NamespaceNotFound") + { + } + } + + private static void SeedHttpFixtures(CannedHttpMessageHandler handler) + { + var feedUri = new Uri("https://chromereleases.googleblog.com/atom.xml?max-results=50&start-index=1&redirect=false"); + var detailUri = new Uri("https://chromereleases.googleblog.com/2024/09/stable-channel-update-for-desktop.html"); + + handler.AddTextResponse(feedUri, ReadFixture("chromium-feed.xml"), "application/atom+xml"); + handler.AddTextResponse(detailUri, ReadFixture("chromium-detail.html"), "text/html"); + } + + private static string ReadFixture(string filename) + { + var path = ResolveFixturePath(filename); + return File.ReadAllText(path); + } + + private static string ResolveFixturePath(string filename) + { + var baseDirectory = AppContext.BaseDirectory; + var primary = Path.Combine(baseDirectory, "Source", "Vndr", "Chromium", "Fixtures", filename); + if (File.Exists(primary)) + { + return primary; + } + + return Path.Combine(baseDirectory, "Chromium", "Fixtures", filename); + } + + public Task InitializeAsync() => Task.CompletedTask; + + public async Task DisposeAsync() + { + foreach (var name in _allocatedDatabases.Distinct(StringComparer.Ordinal)) + { + await DropDatabaseAsync(name); + } + } +} diff --git a/src/StellaOps.Feedser.Source.Vndr.Chromium.Tests/Chromium/ChromiumMapperTests.cs b/src/StellaOps.Feedser.Source.Vndr.Chromium.Tests/Chromium/ChromiumMapperTests.cs new file mode 100644 index 00000000..ca26fa24 --- /dev/null +++ b/src/StellaOps.Feedser.Source.Vndr.Chromium.Tests/Chromium/ChromiumMapperTests.cs @@ -0,0 +1,47 @@ +using System; +using System.Linq; +using StellaOps.Feedser.Source.Vndr.Chromium; +using StellaOps.Feedser.Source.Vndr.Chromium.Internal; +using Xunit; + +namespace StellaOps.Feedser.Source.Vndr.Chromium.Tests; + +public sealed class ChromiumMapperTests +{ + [Fact] + public void Map_DeduplicatesReferencesAndOrdersDeterministically() + { + var published = new DateTimeOffset(2024, 9, 12, 14, 0, 0, TimeSpan.Zero); + var metadata = new ChromiumDocumentMetadata( + "post-123", + "Stable Channel Update", + new Uri("https://chromium.example/stable-update.html"), + published, + null, + "Security fixes"); + + var dto = ChromiumDto.From( + metadata, + new[] { "CVE-2024-0001" }, + new[] { "windows" }, + new[] { new ChromiumVersionInfo("windows", "stable", "128.0.6613.88") }, + new[] + { + new ChromiumReference("https://chromium.example/ref1", "advisory", "Ref 1"), + new ChromiumReference("https://chromium.example/ref1", "advisory", "Ref 1 duplicate"), + new ChromiumReference("https://chromium.example/ref2", "patch", "Ref 2"), + }); + + var (advisory, _) = ChromiumMapper.Map(dto, VndrChromiumConnectorPlugin.SourceName, published); + + var referenceUrls = advisory.References.Select(r => r.Url).ToArray(); + Assert.Equal( + new[] + { + "https://chromium.example/ref1", + "https://chromium.example/ref2", + "https://chromium.example/stable-update.html", + }, + referenceUrls); + } +} diff --git a/src/StellaOps.Feedser.Source.Vndr.Chromium.Tests/Chromium/Fixtures/chromium-advisory.snapshot.json b/src/StellaOps.Feedser.Source.Vndr.Chromium.Tests/Chromium/Fixtures/chromium-advisory.snapshot.json new file mode 100644 index 00000000..2a54569a --- /dev/null +++ b/src/StellaOps.Feedser.Source.Vndr.Chromium.Tests/Chromium/Fixtures/chromium-advisory.snapshot.json @@ -0,0 +1 @@ +{"advisoryKey":"chromium/post/stable-channel-update-for-desktop","affectedPackages":[{"identifier":"google:chrome","platform":"android","provenance":[{"fieldMask":[],"kind":"document","recordedAt":"2024-09-10T18:00:00+00:00","source":"vndr-chromium","value":"stable-channel-update-for-desktop"}],"statuses":[],"type":"vendor","versionRanges":[{"fixedVersion":"128.0.6613.89","introducedVersion":null,"lastAffectedVersion":null,"primitives":{"evr":null,"hasVendorExtensions":true,"nevra":null,"semVer":null,"vendorExtensions":{"chromium.channel":"stable","chromium.platform":"android","chromium.version.raw":"128.0.6613.89","chromium.version.normalized":"128.0.6613.89","chromium.version.major":"128","chromium.version.minor":"0","chromium.version.build":"6613","chromium.version.patch":"89"}},"provenance":{"fieldMask":[],"kind":"document","recordedAt":"2024-09-10T18:00:00+00:00","source":"vndr-chromium","value":"stable-channel-update-for-desktop"},"rangeExpression":null,"rangeKind":"vendor"}]},{"identifier":"google:chrome","platform":"linux","provenance":[{"fieldMask":[],"kind":"document","recordedAt":"2024-09-10T18:00:00+00:00","source":"vndr-chromium","value":"stable-channel-update-for-desktop"}],"statuses":[],"type":"vendor","versionRanges":[{"fixedVersion":"128.0.6613.137","introducedVersion":null,"lastAffectedVersion":null,"primitives":{"evr":null,"hasVendorExtensions":true,"nevra":null,"semVer":null,"vendorExtensions":{"chromium.channel":"stable","chromium.platform":"linux","chromium.version.raw":"128.0.6613.137","chromium.version.normalized":"128.0.6613.137","chromium.version.major":"128","chromium.version.minor":"0","chromium.version.build":"6613","chromium.version.patch":"137"}},"provenance":{"fieldMask":[],"kind":"document","recordedAt":"2024-09-10T18:00:00+00:00","source":"vndr-chromium","value":"stable-channel-update-for-desktop"},"rangeExpression":null,"rangeKind":"vendor"}]},{"identifier":"google:chrome","platform":"windows-mac","provenance":[{"fieldMask":[],"kind":"document","recordedAt":"2024-09-10T18:00:00+00:00","source":"vndr-chromium","value":"stable-channel-update-for-desktop"}],"statuses":[],"type":"vendor","versionRanges":[{"fixedVersion":"128.0.6613.138","introducedVersion":null,"lastAffectedVersion":null,"primitives":{"evr":null,"hasVendorExtensions":true,"nevra":null,"semVer":null,"vendorExtensions":{"chromium.channel":"stable","chromium.platform":"windows-mac","chromium.version.raw":"128.0.6613.138","chromium.version.normalized":"128.0.6613.138","chromium.version.major":"128","chromium.version.minor":"0","chromium.version.build":"6613","chromium.version.patch":"138"}},"provenance":{"fieldMask":[],"kind":"document","recordedAt":"2024-09-10T18:00:00+00:00","source":"vndr-chromium","value":"stable-channel-update-for-desktop"},"rangeExpression":null,"rangeKind":"vendor"}]},{"identifier":"google:chrome:extended-stable","platform":"windows-mac","provenance":[{"fieldMask":[],"kind":"document","recordedAt":"2024-09-10T18:00:00+00:00","source":"vndr-chromium","value":"stable-channel-update-for-desktop"}],"statuses":[],"type":"vendor","versionRanges":[{"fixedVersion":"128.0.6613.138","introducedVersion":null,"lastAffectedVersion":null,"primitives":{"evr":null,"hasVendorExtensions":true,"nevra":null,"semVer":null,"vendorExtensions":{"chromium.channel":"extended-stable","chromium.platform":"windows-mac","chromium.version.raw":"128.0.6613.138","chromium.version.normalized":"128.0.6613.138","chromium.version.major":"128","chromium.version.minor":"0","chromium.version.build":"6613","chromium.version.patch":"138"}},"provenance":{"fieldMask":[],"kind":"document","recordedAt":"2024-09-10T18:00:00+00:00","source":"vndr-chromium","value":"stable-channel-update-for-desktop"},"rangeExpression":null,"rangeKind":"vendor"}]}],"aliases":["CHROMIUM-POST:2024-09-10","CHROMIUM-POST:stable-channel-update-for-desktop","CVE-2024-12345","CVE-2024-22222"],"cvssMetrics":[],"exploitKnown":false,"language":"en","modified":"2024-09-10T17:45:00+00:00","provenance":[{"fieldMask":[],"kind":"document","recordedAt":"2024-09-10T18:00:00+00:00","source":"vndr-chromium","value":"stable-channel-update-for-desktop"}],"published":"2024-09-10T17:30:00+00:00","references":[{"kind":"advisory","provenance":{"fieldMask":[],"kind":"document","recordedAt":"2024-09-10T18:00:00+00:00","source":"vndr-chromium","value":"stable-channel-update-for-desktop"},"sourceTag":"chromium-blog","summary":null,"url":"https://chromereleases.googleblog.com/2024/09/stable-channel-update-for-desktop.html"},{"kind":"changelog","provenance":{"fieldMask":[],"kind":"document","recordedAt":"2024-09-10T18:00:00+00:00","source":"vndr-chromium","value":"stable-channel-update-for-desktop"},"sourceTag":"changelog","summary":"log","url":"https://chromium.googlesource.com/chromium/src/+log/128.0.6613.120..128.0.6613.138"},{"kind":"doc","provenance":{"fieldMask":[],"kind":"document","recordedAt":"2024-09-10T18:00:00+00:00","source":"vndr-chromium","value":"stable-channel-update-for-desktop"},"sourceTag":"doc","summary":"security page","url":"https://chromium.org/Home/chromium-security"},{"kind":"bug","provenance":{"fieldMask":[],"kind":"document","recordedAt":"2024-09-10T18:00:00+00:00","source":"vndr-chromium","value":"stable-channel-update-for-desktop"},"sourceTag":"bug","summary":"issue tracker","url":"https://issues.chromium.org/issues/123456789"}],"severity":null,"summary":"Stable channel update rolling out to Windows, macOS, Linux.","title":"Stable Channel Update for Desktop"} \ No newline at end of file diff --git a/src/StellaOps.Feedser.Source.Vndr.Chromium.Tests/Chromium/Fixtures/chromium-detail.html b/src/StellaOps.Feedser.Source.Vndr.Chromium.Tests/Chromium/Fixtures/chromium-detail.html new file mode 100644 index 00000000..dc970d80 --- /dev/null +++ b/src/StellaOps.Feedser.Source.Vndr.Chromium.Tests/Chromium/Fixtures/chromium-detail.html @@ -0,0 +1,21 @@ + + + + + Stable Channel Update for Desktop + + +
      +

      The Stable channel has been updated to 128.0.6613.138 for Windows and macOS, and 128.0.6613.137 for Linux. A full list of changes in this build is available in the log.

      +

      The Extended Stable channel has been updated to 128.0.6613.138 for Windows and Mac and will roll out over the coming days.

      +

      The team is also rolling out Chrome 128.0.6613.89 to Android.

      +

      Security Fixes and Rewards

      +

      We would like to thank all security researchers who worked with us during the development cycle.

      +
        +
      • CVE-2024-12345: Use after free in Blink.
        • +
      • CVE-2024-22222: Heap buffer overflow in GPU.
        • +
      +

      For details see the issue tracker and the security page.

      +
      + + diff --git a/src/StellaOps.Feedser.Source.Vndr.Chromium.Tests/Chromium/Fixtures/chromium-feed.xml b/src/StellaOps.Feedser.Source.Vndr.Chromium.Tests/Chromium/Fixtures/chromium-feed.xml new file mode 100644 index 00000000..55d0d3cd --- /dev/null +++ b/src/StellaOps.Feedser.Source.Vndr.Chromium.Tests/Chromium/Fixtures/chromium-feed.xml @@ -0,0 +1,16 @@ + + + tag:blogger.com,1999:blog-8982037438137564684 + 2024-09-10T18:00:00Z + Google Chrome Releases + + + tag:blogger.com,1999:blog-8982037438137564684.post-123456789 + 2024-09-10T17:30:00Z + 2024-09-10T17:45:00Z + Stable Channel Update for Desktop + Stable channel update rolling out to Windows, macOS, Linux. + + + + diff --git a/src/StellaOps.Feedser.Source.Vndr.Chromium.Tests/StellaOps.Feedser.Source.Vndr.Chromium.Tests.csproj b/src/StellaOps.Feedser.Source.Vndr.Chromium.Tests/StellaOps.Feedser.Source.Vndr.Chromium.Tests.csproj new file mode 100644 index 00000000..887bec82 --- /dev/null +++ b/src/StellaOps.Feedser.Source.Vndr.Chromium.Tests/StellaOps.Feedser.Source.Vndr.Chromium.Tests.csproj @@ -0,0 +1,18 @@ + + + net10.0 + enable + enable + + + + + + + + + + + + + diff --git a/src/StellaOps.Feedser.Source.Vndr.Chromium/AGENTS.md b/src/StellaOps.Feedser.Source.Vndr.Chromium/AGENTS.md new file mode 100644 index 00000000..a87e00fb --- /dev/null +++ b/src/StellaOps.Feedser.Source.Vndr.Chromium/AGENTS.md @@ -0,0 +1,28 @@ +# AGENTS +## Role +Chromium/Chrome vendor feed connector parsing Stable Channel Update posts; authoritative vendor context for Chrome/Chromium versions and CVE lists; maps fixed versions as affected ranges. +## Scope +- Crawl Chrome Releases blog list; window by publish date; fetch detail posts; identify "Stable Channel Update" and security fix sections. +- Validate HTML; extract version trains, platform notes (Windows/macOS/Linux/Android), CVEs, acknowledgements; map fixed versions. +- Persist raw docs and maintain source_state cursor; idempotent mapping. +## Participants +- Source.Common (HTTP, HTML helpers, validators). +- Storage.Mongo (document, dto, advisory, alias, affected, reference, psirt_flags, source_state). +- Models (canonical; affected ranges by product/version). +- Core/WebService (jobs: source:chromium:fetch|parse|map). +- Merge engine (later) to respect vendor PSIRT precedence for Chrome. +## Interfaces & contracts +- Aliases: CHROMIUM-POST: plus CVE ids. +- Affected: Vendor=Google, Product=Chrome/Chromium (platform tags), Type=vendor; Versions indicate introduced? (often unknown) and fixed (for example 127.0.6533.88); tags mark platforms. +- References: advisory (post URL), release notes, bug links; kind set appropriately. +- Provenance: method=parser; value=post slug; recordedAt=fetch time. +## In/Out of scope +In: vendor advisory mapping, fixed version emission per platform, psirt_flags vendor context. +Out: OS distro packaging semantics; bug bounty details beyond references. +## Observability & security expectations +- Metrics: SourceDiagnostics exports the shared `feedser.source.http.*` counters/histograms tagged `feedser.source=chromium`, enabling dashboards to observe fetch volumes, parse failures, and map affected counts via tag filters. +- Logs: post slugs, version extracted, platform coverage, timing; allowlist blog host. +## Tests +- Author and review coverage in `../StellaOps.Feedser.Source.Vndr.Chromium.Tests`. +- Shared fixtures (e.g., `MongoIntegrationFixture`, `ConnectorTestHarness`) live in `../StellaOps.Feedser.Testing`. +- Keep fixtures deterministic; match new cases to real-world advisories or regression scenarios. diff --git a/src/StellaOps.Feedser.Source.Vndr.Chromium/ChromiumConnector.cs b/src/StellaOps.Feedser.Source.Vndr.Chromium/ChromiumConnector.cs new file mode 100644 index 00000000..2e432fc3 --- /dev/null +++ b/src/StellaOps.Feedser.Source.Vndr.Chromium/ChromiumConnector.cs @@ -0,0 +1,366 @@ +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Text.Json; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Options; +using MongoDB.Bson; +using MongoDB.Bson.IO; +using StellaOps.Feedser.Models; +using StellaOps.Feedser.Source.Common; +using StellaOps.Feedser.Source.Common.Fetch; +using StellaOps.Feedser.Source.Common.Json; +using StellaOps.Feedser.Source.Vndr.Chromium.Configuration; +using StellaOps.Feedser.Source.Vndr.Chromium.Internal; +using StellaOps.Feedser.Storage.Mongo; +using StellaOps.Feedser.Storage.Mongo.Advisories; +using StellaOps.Feedser.Storage.Mongo.Documents; +using StellaOps.Feedser.Storage.Mongo.Dtos; +using StellaOps.Feedser.Storage.Mongo.PsirtFlags; +using StellaOps.Plugin; +using Json.Schema; + +namespace StellaOps.Feedser.Source.Vndr.Chromium; + +public sealed class ChromiumConnector : IFeedConnector +{ + private static readonly JsonSchema Schema = ChromiumSchemaProvider.Schema; + private static readonly JsonSerializerOptions SerializerOptions = new() + { + PropertyNamingPolicy = JsonNamingPolicy.CamelCase, + DefaultIgnoreCondition = System.Text.Json.Serialization.JsonIgnoreCondition.WhenWritingNull, + }; + + private readonly ChromiumFeedLoader _feedLoader; + private readonly SourceFetchService _fetchService; + private readonly RawDocumentStorage _rawDocumentStorage; + private readonly IDocumentStore _documentStore; + private readonly IDtoStore _dtoStore; + private readonly IAdvisoryStore _advisoryStore; + private readonly IPsirtFlagStore _psirtFlagStore; + private readonly ISourceStateRepository _stateRepository; + private readonly IJsonSchemaValidator _schemaValidator; + private readonly ChromiumOptions _options; + private readonly TimeProvider _timeProvider; + private readonly ChromiumDiagnostics _diagnostics; + private readonly ILogger _logger; + + public ChromiumConnector( + ChromiumFeedLoader feedLoader, + SourceFetchService fetchService, + RawDocumentStorage rawDocumentStorage, + IDocumentStore documentStore, + IDtoStore dtoStore, + IAdvisoryStore advisoryStore, + IPsirtFlagStore psirtFlagStore, + ISourceStateRepository stateRepository, + IJsonSchemaValidator schemaValidator, + IOptions options, + TimeProvider? timeProvider, + ChromiumDiagnostics diagnostics, + ILogger logger) + { + _feedLoader = feedLoader ?? throw new ArgumentNullException(nameof(feedLoader)); + _fetchService = fetchService ?? throw new ArgumentNullException(nameof(fetchService)); + _rawDocumentStorage = rawDocumentStorage ?? throw new ArgumentNullException(nameof(rawDocumentStorage)); + _documentStore = documentStore ?? throw new ArgumentNullException(nameof(documentStore)); + _dtoStore = dtoStore ?? throw new ArgumentNullException(nameof(dtoStore)); + _advisoryStore = advisoryStore ?? throw new ArgumentNullException(nameof(advisoryStore)); + _psirtFlagStore = psirtFlagStore ?? throw new ArgumentNullException(nameof(psirtFlagStore)); + _stateRepository = stateRepository ?? throw new ArgumentNullException(nameof(stateRepository)); + _schemaValidator = schemaValidator ?? throw new ArgumentNullException(nameof(schemaValidator)); + _options = options?.Value ?? throw new ArgumentNullException(nameof(options)); + _options.Validate(); + _timeProvider = timeProvider ?? TimeProvider.System; + _diagnostics = diagnostics ?? throw new ArgumentNullException(nameof(diagnostics)); + _logger = logger ?? throw new ArgumentNullException(nameof(logger)); + } + + public string SourceName => VndrChromiumConnectorPlugin.SourceName; + + public async Task FetchAsync(IServiceProvider services, CancellationToken cancellationToken) + { + var cursor = await GetCursorAsync(cancellationToken).ConfigureAwait(false); + var now = _timeProvider.GetUtcNow(); + var (windowStart, windowEnd) = CalculateWindow(cursor, now); + ProvenanceDiagnostics.ReportResumeWindow(SourceName, windowStart, _logger); + + IReadOnlyList feedEntries; + _diagnostics.FetchAttempt(); + try + { + feedEntries = await _feedLoader.LoadAsync(windowStart, windowEnd, cancellationToken).ConfigureAwait(false); + } + catch (Exception ex) + { + _logger.LogError(ex, "Chromium feed load failed {Start}-{End}", windowStart, windowEnd); + _diagnostics.FetchFailure(); + await _stateRepository.MarkFailureAsync(SourceName, now, TimeSpan.FromMinutes(10), ex.Message, cancellationToken).ConfigureAwait(false); + throw; + } + + var fetchCache = new Dictionary(cursor.FetchCache, StringComparer.Ordinal); + var touchedResources = new HashSet(StringComparer.Ordinal); + + var candidates = feedEntries + .Where(static entry => entry.IsSecurityUpdate()) + .OrderBy(static entry => entry.Published) + .ToArray(); + + if (candidates.Length == 0) + { + var untouched = cursor + .WithLastPublished(cursor.LastPublished ?? windowEnd) + .WithFetchCache(fetchCache); + await UpdateCursorAsync(untouched, cancellationToken).ConfigureAwait(false); + return; + } + + var pendingDocuments = cursor.PendingDocuments.ToList(); + var maxPublished = cursor.LastPublished; + + foreach (var entry in candidates) + { + try + { + var cacheKey = entry.DetailUri.ToString(); + touchedResources.Add(cacheKey); + + var metadata = ChromiumDocumentMetadata.CreateMetadata(entry.PostId, entry.Title, entry.Published, entry.Updated, entry.Summary); + var request = new SourceFetchRequest(ChromiumOptions.HttpClientName, SourceName, entry.DetailUri) + { + Metadata = metadata, + AcceptHeaders = new[] { "text/html", "application/xhtml+xml", "text/plain;q=0.5" }, + }; + + var result = await _fetchService.FetchAsync(request, cancellationToken).ConfigureAwait(false); + if (!result.IsSuccess || result.Document is null) + { + continue; + } + + if (cursor.TryGetFetchCache(cacheKey, out var cached) && string.Equals(cached.Sha256, result.Document.Sha256, StringComparison.OrdinalIgnoreCase)) + { + _diagnostics.FetchUnchanged(); + fetchCache[cacheKey] = new ChromiumFetchCacheEntry(result.Document.Sha256); + await _documentStore.UpdateStatusAsync(result.Document.Id, DocumentStatuses.Mapped, cancellationToken).ConfigureAwait(false); + + if (!maxPublished.HasValue || entry.Published > maxPublished) + { + maxPublished = entry.Published; + } + + continue; + } + + _diagnostics.FetchDocument(); + if (!pendingDocuments.Contains(result.Document.Id)) + { + pendingDocuments.Add(result.Document.Id); + } + + if (!maxPublished.HasValue || entry.Published > maxPublished) + { + maxPublished = entry.Published; + } + + fetchCache[cacheKey] = new ChromiumFetchCacheEntry(result.Document.Sha256); + } + catch (Exception ex) + { + _logger.LogError(ex, "Chromium fetch failed for {Uri}", entry.DetailUri); + _diagnostics.FetchFailure(); + await _stateRepository.MarkFailureAsync(SourceName, now, TimeSpan.FromMinutes(5), ex.Message, cancellationToken).ConfigureAwait(false); + throw; + } + } + + if (touchedResources.Count > 0) + { + var keysToRemove = fetchCache.Keys.Where(key => !touchedResources.Contains(key)).ToArray(); + foreach (var key in keysToRemove) + { + fetchCache.Remove(key); + } + } + + var updatedCursor = cursor + .WithPendingDocuments(pendingDocuments) + .WithPendingMappings(cursor.PendingMappings) + .WithLastPublished(maxPublished ?? cursor.LastPublished ?? windowEnd) + .WithFetchCache(fetchCache); + + await UpdateCursorAsync(updatedCursor, cancellationToken).ConfigureAwait(false); + } + + public async Task ParseAsync(IServiceProvider services, CancellationToken cancellationToken) + { + var cursor = await GetCursorAsync(cancellationToken).ConfigureAwait(false); + if (cursor.PendingDocuments.Count == 0) + { + return; + } + + var pendingDocuments = cursor.PendingDocuments.ToList(); + var pendingMappings = cursor.PendingMappings.ToList(); + + foreach (var documentId in cursor.PendingDocuments) + { + var document = await _documentStore.FindAsync(documentId, cancellationToken).ConfigureAwait(false); + if (document is null) + { + pendingDocuments.Remove(documentId); + pendingMappings.Remove(documentId); + continue; + } + + if (!document.GridFsId.HasValue) + { + _logger.LogWarning("Chromium document {DocumentId} missing GridFS payload", document.Id); + await _documentStore.UpdateStatusAsync(document.Id, DocumentStatuses.Failed, cancellationToken).ConfigureAwait(false); + pendingDocuments.Remove(documentId); + pendingMappings.Remove(documentId); + continue; + } + + ChromiumDto dto; + try + { + var metadata = ChromiumDocumentMetadata.FromDocument(document); + var content = await _rawDocumentStorage.DownloadAsync(document.GridFsId.Value, cancellationToken).ConfigureAwait(false); + var html = Encoding.UTF8.GetString(content); + dto = ChromiumParser.Parse(html, metadata); + } + catch (Exception ex) + { + _logger.LogError(ex, "Chromium parse failed for {Uri}", document.Uri); + _diagnostics.ParseFailure(); + await _documentStore.UpdateStatusAsync(document.Id, DocumentStatuses.Failed, cancellationToken).ConfigureAwait(false); + pendingDocuments.Remove(documentId); + pendingMappings.Remove(documentId); + continue; + } + + var json = JsonSerializer.Serialize(dto, SerializerOptions); + using var jsonDocument = JsonDocument.Parse(json); + try + { + _schemaValidator.Validate(jsonDocument, Schema, dto.PostId); + } + catch (StellaOps.Feedser.Source.Common.Json.JsonSchemaValidationException ex) + { + _logger.LogError(ex, "Chromium schema validation failed for {DocumentId}", document.Id); + await _documentStore.UpdateStatusAsync(document.Id, DocumentStatuses.Failed, cancellationToken).ConfigureAwait(false); + pendingDocuments.Remove(documentId); + pendingMappings.Remove(documentId); + continue; + } + + var payload = BsonDocument.Parse(json); + var existingDto = await _dtoStore.FindByDocumentIdAsync(document.Id, cancellationToken).ConfigureAwait(false); + var validatedAt = _timeProvider.GetUtcNow(); + + var dtoRecord = existingDto is null + ? new DtoRecord(Guid.NewGuid(), document.Id, SourceName, "chromium.post.v1", payload, validatedAt) + : existingDto with + { + Payload = payload, + SchemaVersion = "chromium.post.v1", + ValidatedAt = validatedAt, + }; + + await _dtoStore.UpsertAsync(dtoRecord, cancellationToken).ConfigureAwait(false); + await _documentStore.UpdateStatusAsync(document.Id, DocumentStatuses.PendingMap, cancellationToken).ConfigureAwait(false); + _diagnostics.ParseSuccess(); + + pendingDocuments.Remove(documentId); + if (!pendingMappings.Contains(documentId)) + { + pendingMappings.Add(documentId); + } + } + + var updatedCursor = cursor + .WithPendingDocuments(pendingDocuments) + .WithPendingMappings(pendingMappings); + + await UpdateCursorAsync(updatedCursor, cancellationToken).ConfigureAwait(false); + } + + public async Task MapAsync(IServiceProvider services, CancellationToken cancellationToken) + { + var cursor = await GetCursorAsync(cancellationToken).ConfigureAwait(false); + if (cursor.PendingMappings.Count == 0) + { + return; + } + + var pendingMappings = cursor.PendingMappings.ToList(); + + foreach (var documentId in cursor.PendingMappings) + { + var dtoRecord = await _dtoStore.FindByDocumentIdAsync(documentId, cancellationToken).ConfigureAwait(false); + var document = await _documentStore.FindAsync(documentId, cancellationToken).ConfigureAwait(false); + + if (dtoRecord is null || document is null) + { + pendingMappings.Remove(documentId); + continue; + } + + var json = dtoRecord.Payload.ToJson(new JsonWriterSettings { OutputMode = JsonOutputMode.RelaxedExtendedJson }); + var dto = JsonSerializer.Deserialize(json, SerializerOptions); + if (dto is null) + { + _logger.LogWarning("Chromium DTO deserialization failed for {DocumentId}", documentId); + await _documentStore.UpdateStatusAsync(document.Id, DocumentStatuses.Failed, cancellationToken).ConfigureAwait(false); + pendingMappings.Remove(documentId); + continue; + } + + var recordedAt = _timeProvider.GetUtcNow(); + var (advisory, flag) = ChromiumMapper.Map(dto, SourceName, recordedAt); + + await _advisoryStore.UpsertAsync(advisory, cancellationToken).ConfigureAwait(false); + await _psirtFlagStore.UpsertAsync(flag, cancellationToken).ConfigureAwait(false); + await _documentStore.UpdateStatusAsync(document.Id, DocumentStatuses.Mapped, cancellationToken).ConfigureAwait(false); + _diagnostics.MapSuccess(); + + pendingMappings.Remove(documentId); + } + + var updatedCursor = cursor.WithPendingMappings(pendingMappings); + await UpdateCursorAsync(updatedCursor, cancellationToken).ConfigureAwait(false); + } + + private async Task GetCursorAsync(CancellationToken cancellationToken) + { + var record = await _stateRepository.TryGetAsync(SourceName, cancellationToken).ConfigureAwait(false); + return ChromiumCursor.FromBsonDocument(record?.Cursor); + } + + private async Task UpdateCursorAsync(ChromiumCursor cursor, CancellationToken cancellationToken) + { + var completedAt = _timeProvider.GetUtcNow(); + await _stateRepository.UpdateCursorAsync(SourceName, cursor.ToBsonDocument(), completedAt, cancellationToken).ConfigureAwait(false); + } + + private (DateTimeOffset start, DateTimeOffset end) CalculateWindow(ChromiumCursor cursor, DateTimeOffset now) + { + var lastPublished = cursor.LastPublished ?? now - _options.InitialBackfill; + var start = lastPublished - _options.WindowOverlap; + var backfill = now - _options.InitialBackfill; + if (start < backfill) + { + start = backfill; + } + + var end = now; + if (end <= start) + { + end = start.AddHours(1); + } + + return (start, end); + } +} diff --git a/src/StellaOps.Feedser.Source.Vndr.Chromium/ChromiumConnectorPlugin.cs b/src/StellaOps.Feedser.Source.Vndr.Chromium/ChromiumConnectorPlugin.cs new file mode 100644 index 00000000..8cb72d05 --- /dev/null +++ b/src/StellaOps.Feedser.Source.Vndr.Chromium/ChromiumConnectorPlugin.cs @@ -0,0 +1,20 @@ +using System; +using Microsoft.Extensions.DependencyInjection; +using StellaOps.Plugin; + +namespace StellaOps.Feedser.Source.Vndr.Chromium; + +public sealed class VndrChromiumConnectorPlugin : IConnectorPlugin +{ + public const string SourceName = "vndr-chromium"; + + public string Name => SourceName; + + public bool IsAvailable(IServiceProvider services) => services.GetService() is not null; + + public IFeedConnector Create(IServiceProvider services) + { + ArgumentNullException.ThrowIfNull(services); + return services.GetRequiredService(); + } +} diff --git a/src/StellaOps.Feedser.Source.Vndr.Chromium/ChromiumDiagnostics.cs b/src/StellaOps.Feedser.Source.Vndr.Chromium/ChromiumDiagnostics.cs new file mode 100644 index 00000000..cf5c1de6 --- /dev/null +++ b/src/StellaOps.Feedser.Source.Vndr.Chromium/ChromiumDiagnostics.cs @@ -0,0 +1,69 @@ +using System.Diagnostics.Metrics; + +namespace StellaOps.Feedser.Source.Vndr.Chromium; + +public sealed class ChromiumDiagnostics : IDisposable +{ + public const string MeterName = "StellaOps.Feedser.Source.Vndr.Chromium"; + public const string MeterVersion = "1.0.0"; + + private readonly Meter _meter; + private readonly Counter _fetchAttempts; + private readonly Counter _fetchDocuments; + private readonly Counter _fetchFailures; + private readonly Counter _fetchUnchanged; + private readonly Counter _parseSuccess; + private readonly Counter _parseFailures; + private readonly Counter _mapSuccess; + + public ChromiumDiagnostics() + { + _meter = new Meter(MeterName, MeterVersion); + _fetchAttempts = _meter.CreateCounter( + name: "chromium.fetch.attempts", + unit: "operations", + description: "Number of Chromium fetch operations executed."); + _fetchDocuments = _meter.CreateCounter( + name: "chromium.fetch.documents", + unit: "documents", + description: "Count of Chromium advisory documents fetched successfully."); + _fetchFailures = _meter.CreateCounter( + name: "chromium.fetch.failures", + unit: "operations", + description: "Count of Chromium fetch failures."); + _fetchUnchanged = _meter.CreateCounter( + name: "chromium.fetch.unchanged", + unit: "documents", + description: "Count of Chromium documents skipped due to unchanged content."); + _parseSuccess = _meter.CreateCounter( + name: "chromium.parse.success", + unit: "documents", + description: "Count of Chromium documents parsed successfully."); + _parseFailures = _meter.CreateCounter( + name: "chromium.parse.failures", + unit: "documents", + description: "Count of Chromium documents that failed to parse."); + _mapSuccess = _meter.CreateCounter( + name: "chromium.map.success", + unit: "advisories", + description: "Count of Chromium advisories mapped successfully."); + } + + public void FetchAttempt() => _fetchAttempts.Add(1); + + public void FetchDocument() => _fetchDocuments.Add(1); + + public void FetchFailure() => _fetchFailures.Add(1); + + public void FetchUnchanged() => _fetchUnchanged.Add(1); + + public void ParseSuccess() => _parseSuccess.Add(1); + + public void ParseFailure() => _parseFailures.Add(1); + + public void MapSuccess() => _mapSuccess.Add(1); + + public Meter Meter => _meter; + + public void Dispose() => _meter.Dispose(); +} diff --git a/src/StellaOps.Feedser.Source.Vndr.Chromium/ChromiumServiceCollectionExtensions.cs b/src/StellaOps.Feedser.Source.Vndr.Chromium/ChromiumServiceCollectionExtensions.cs new file mode 100644 index 00000000..16f7351d --- /dev/null +++ b/src/StellaOps.Feedser.Source.Vndr.Chromium/ChromiumServiceCollectionExtensions.cs @@ -0,0 +1,37 @@ +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Options; +using StellaOps.Feedser.Source.Common.Http; +using StellaOps.Feedser.Source.Vndr.Chromium.Configuration; +using StellaOps.Feedser.Source.Vndr.Chromium.Internal; + +namespace StellaOps.Feedser.Source.Vndr.Chromium; + +public static class ChromiumServiceCollectionExtensions +{ + public static IServiceCollection AddChromiumConnector(this IServiceCollection services, Action configure) + { + ArgumentNullException.ThrowIfNull(services); + ArgumentNullException.ThrowIfNull(configure); + + services.AddOptions() + .Configure(configure) + .PostConfigure(static opts => opts.Validate()); + + services.AddSingleton(static sp => sp.GetRequiredService>().Value); + + services.AddSourceHttpClient(ChromiumOptions.HttpClientName, static (sp, clientOptions) => + { + var options = sp.GetRequiredService>().Value; + clientOptions.BaseAddress = new Uri(options.FeedUri.GetLeftPart(UriPartial.Authority)); + clientOptions.Timeout = TimeSpan.FromSeconds(20); + clientOptions.UserAgent = "StellaOps.Feedser.VndrChromium/1.0"; + clientOptions.AllowedHosts.Clear(); + clientOptions.AllowedHosts.Add(options.FeedUri.Host); + }); + + services.AddSingleton(); + services.AddTransient(); + services.AddTransient(); + return services; + } +} diff --git a/src/StellaOps.Feedser.Source.Vndr.Chromium/Configuration/ChromiumOptions.cs b/src/StellaOps.Feedser.Source.Vndr.Chromium/Configuration/ChromiumOptions.cs new file mode 100644 index 00000000..08619672 --- /dev/null +++ b/src/StellaOps.Feedser.Source.Vndr.Chromium/Configuration/ChromiumOptions.cs @@ -0,0 +1,44 @@ +namespace StellaOps.Feedser.Source.Vndr.Chromium.Configuration; + +public sealed class ChromiumOptions +{ + public const string HttpClientName = "source-vndr-chromium"; + + public Uri FeedUri { get; set; } = new("https://chromereleases.googleblog.com/atom.xml"); + + public TimeSpan InitialBackfill { get; set; } = TimeSpan.FromDays(30); + + public TimeSpan WindowOverlap { get; set; } = TimeSpan.FromDays(2); + + public int MaxFeedPages { get; set; } = 4; + + public int MaxEntriesPerPage { get; set; } = 50; + + public void Validate() + { + if (FeedUri is null || !FeedUri.IsAbsoluteUri) + { + throw new ArgumentException("FeedUri must be an absolute URI.", nameof(FeedUri)); + } + + if (InitialBackfill <= TimeSpan.Zero) + { + throw new ArgumentException("InitialBackfill must be positive.", nameof(InitialBackfill)); + } + + if (WindowOverlap < TimeSpan.Zero) + { + throw new ArgumentException("WindowOverlap cannot be negative.", nameof(WindowOverlap)); + } + + if (MaxFeedPages <= 0) + { + throw new ArgumentException("MaxFeedPages must be positive.", nameof(MaxFeedPages)); + } + + if (MaxEntriesPerPage <= 0 || MaxEntriesPerPage > 100) + { + throw new ArgumentException("MaxEntriesPerPage must be between 1 and 100.", nameof(MaxEntriesPerPage)); + } + } +} diff --git a/src/StellaOps.Feedser.Source.Vndr.Chromium/Internal/ChromiumCursor.cs b/src/StellaOps.Feedser.Source.Vndr.Chromium/Internal/ChromiumCursor.cs new file mode 100644 index 00000000..622ae605 --- /dev/null +++ b/src/StellaOps.Feedser.Source.Vndr.Chromium/Internal/ChromiumCursor.cs @@ -0,0 +1,143 @@ +using System.Collections.Generic; +using System.Linq; +using MongoDB.Bson; + +namespace StellaOps.Feedser.Source.Vndr.Chromium.Internal; + +internal sealed record ChromiumCursor( + DateTimeOffset? LastPublished, + IReadOnlyCollection PendingDocuments, + IReadOnlyCollection PendingMappings, + IReadOnlyDictionary FetchCache) +{ + public static ChromiumCursor Empty { get; } = new(null, Array.Empty(), Array.Empty(), new Dictionary(StringComparer.Ordinal)); + + public BsonDocument ToBsonDocument() + { + var document = new BsonDocument(); + if (LastPublished.HasValue) + { + document["lastPublished"] = LastPublished.Value.UtcDateTime; + } + + document["pendingDocuments"] = new BsonArray(PendingDocuments.Select(id => id.ToString())); + document["pendingMappings"] = new BsonArray(PendingMappings.Select(id => id.ToString())); + + if (FetchCache.Count > 0) + { + var cacheDocument = new BsonDocument(); + foreach (var (key, entry) in FetchCache) + { + cacheDocument[key] = entry.ToBson(); + } + + document["fetchCache"] = cacheDocument; + } + + return document; + } + + public static ChromiumCursor FromBsonDocument(BsonDocument? document) + { + if (document is null || document.ElementCount == 0) + { + return Empty; + } + + DateTimeOffset? lastPublished = null; + if (document.TryGetValue("lastPublished", out var lastPublishedValue)) + { + lastPublished = ReadDateTime(lastPublishedValue); + } + + var pendingDocuments = ReadGuidArray(document, "pendingDocuments"); + var pendingMappings = ReadGuidArray(document, "pendingMappings"); + var fetchCache = ReadFetchCache(document); + + return new ChromiumCursor(lastPublished, pendingDocuments, pendingMappings, fetchCache); + } + + public ChromiumCursor WithLastPublished(DateTimeOffset? lastPublished) + => this with { LastPublished = lastPublished?.ToUniversalTime() }; + + public ChromiumCursor WithPendingDocuments(IEnumerable ids) + => this with { PendingDocuments = ids?.Distinct().ToArray() ?? Array.Empty() }; + + public ChromiumCursor WithPendingMappings(IEnumerable ids) + => this with { PendingMappings = ids?.Distinct().ToArray() ?? Array.Empty() }; + + public ChromiumCursor WithFetchCache(IDictionary cache) + => this with { FetchCache = cache is null ? new Dictionary(StringComparer.Ordinal) : new Dictionary(cache, StringComparer.Ordinal) }; + + public bool TryGetFetchCache(string key, out ChromiumFetchCacheEntry entry) + => FetchCache.TryGetValue(key, out entry); + + private static DateTimeOffset? ReadDateTime(BsonValue value) + { + return value.BsonType switch + { + BsonType.DateTime => DateTime.SpecifyKind(value.ToUniversalTime(), DateTimeKind.Utc), + BsonType.String when DateTimeOffset.TryParse(value.AsString, out var parsed) => parsed.ToUniversalTime(), + _ => null, + }; + } + + private static IReadOnlyCollection ReadGuidArray(BsonDocument document, string field) + { + if (!document.TryGetValue(field, out var value) || value is not BsonArray array) + { + return Array.Empty(); + } + + var list = new List(array.Count); + foreach (var element in array) + { + if (Guid.TryParse(element.ToString(), out var guid)) + { + list.Add(guid); + } + } + + return list; + } + + private static IReadOnlyDictionary ReadFetchCache(BsonDocument document) + { + if (!document.TryGetValue("fetchCache", out var value) || value is not BsonDocument cacheDocument) + { + return new Dictionary(StringComparer.Ordinal); + } + + var dictionary = new Dictionary(StringComparer.Ordinal); + foreach (var element in cacheDocument.Elements) + { + if (element.Value is BsonDocument entryDocument) + { + dictionary[element.Name] = ChromiumFetchCacheEntry.FromBson(entryDocument); + } + } + + return dictionary; + } +} + +internal sealed record ChromiumFetchCacheEntry(string Sha256) +{ + public static ChromiumFetchCacheEntry Empty { get; } = new(string.Empty); + + public BsonDocument ToBson() + { + var document = new BsonDocument + { + ["sha256"] = Sha256, + }; + + return document; + } + + public static ChromiumFetchCacheEntry FromBson(BsonDocument document) + { + var sha = document.TryGetValue("sha256", out var shaValue) ? shaValue.AsString : string.Empty; + return new ChromiumFetchCacheEntry(sha); + } +} diff --git a/src/StellaOps.Feedser.Source.Vndr.Chromium/Internal/ChromiumDocumentMetadata.cs b/src/StellaOps.Feedser.Source.Vndr.Chromium/Internal/ChromiumDocumentMetadata.cs new file mode 100644 index 00000000..c1c8cd36 --- /dev/null +++ b/src/StellaOps.Feedser.Source.Vndr.Chromium/Internal/ChromiumDocumentMetadata.cs @@ -0,0 +1,78 @@ +using StellaOps.Feedser.Storage.Mongo.Documents; + +namespace StellaOps.Feedser.Source.Vndr.Chromium.Internal; + +internal sealed record ChromiumDocumentMetadata( + string PostId, + string Title, + Uri DetailUrl, + DateTimeOffset Published, + DateTimeOffset? Updated, + string? Summary) +{ + private const string PostIdKey = "postId"; + private const string TitleKey = "title"; + private const string PublishedKey = "published"; + private const string UpdatedKey = "updated"; + private const string SummaryKey = "summary"; + + public static ChromiumDocumentMetadata FromDocument(DocumentRecord document) + { + ArgumentNullException.ThrowIfNull(document); + + var metadata = document.Metadata ?? throw new InvalidOperationException("Chromium document metadata missing."); + + if (!metadata.TryGetValue(PostIdKey, out var postId) || string.IsNullOrWhiteSpace(postId)) + { + throw new InvalidOperationException("Chromium document metadata missing postId."); + } + + if (!metadata.TryGetValue(TitleKey, out var title) || string.IsNullOrWhiteSpace(title)) + { + throw new InvalidOperationException("Chromium document metadata missing title."); + } + + if (!metadata.TryGetValue(PublishedKey, out var publishedString) || !DateTimeOffset.TryParse(publishedString, out var published)) + { + throw new InvalidOperationException("Chromium document metadata missing published timestamp."); + } + + DateTimeOffset? updated = null; + if (metadata.TryGetValue(UpdatedKey, out var updatedString) && DateTimeOffset.TryParse(updatedString, out var updatedValue)) + { + updated = updatedValue; + } + + metadata.TryGetValue(SummaryKey, out var summary); + + return new ChromiumDocumentMetadata( + postId.Trim(), + title.Trim(), + new Uri(document.Uri, UriKind.Absolute), + published.ToUniversalTime(), + updated?.ToUniversalTime(), + string.IsNullOrWhiteSpace(summary) ? null : summary.Trim()); + } + + public static IReadOnlyDictionary CreateMetadata(string postId, string title, DateTimeOffset published, DateTimeOffset? updated, string? summary) + { + var dictionary = new Dictionary(StringComparer.Ordinal) + { + [PostIdKey] = postId, + [TitleKey] = title, + [PublishedKey] = published.ToUniversalTime().ToString("O"), + }; + + if (updated.HasValue) + { + dictionary[UpdatedKey] = updated.Value.ToUniversalTime().ToString("O"); + } + + if (!string.IsNullOrWhiteSpace(summary)) + { + dictionary[SummaryKey] = summary.Trim(); + } + + return dictionary; + } +} diff --git a/src/StellaOps.Feedser.Source.Vndr.Chromium/Internal/ChromiumDto.cs b/src/StellaOps.Feedser.Source.Vndr.Chromium/Internal/ChromiumDto.cs new file mode 100644 index 00000000..6dacc7a7 --- /dev/null +++ b/src/StellaOps.Feedser.Source.Vndr.Chromium/Internal/ChromiumDto.cs @@ -0,0 +1,39 @@ +using System.Text.Json.Serialization; + +namespace StellaOps.Feedser.Source.Vndr.Chromium.Internal; + +internal sealed record ChromiumDto( + [property: JsonPropertyName("postId")] string PostId, + [property: JsonPropertyName("title")] string Title, + [property: JsonPropertyName("detailUrl")] string DetailUrl, + [property: JsonPropertyName("published")] DateTimeOffset Published, + [property: JsonPropertyName("updated")] DateTimeOffset? Updated, + [property: JsonPropertyName("summary")] string? Summary, + [property: JsonPropertyName("cves")] IReadOnlyList Cves, + [property: JsonPropertyName("platforms")] IReadOnlyList Platforms, + [property: JsonPropertyName("versions")] IReadOnlyList Versions, + [property: JsonPropertyName("references")] IReadOnlyList References) +{ + public static ChromiumDto From(ChromiumDocumentMetadata metadata, IReadOnlyList cves, IReadOnlyList platforms, IReadOnlyList versions, IReadOnlyList references) + => new( + metadata.PostId, + metadata.Title, + metadata.DetailUrl.ToString(), + metadata.Published, + metadata.Updated, + metadata.Summary, + cves, + platforms, + versions, + references); +} + +internal sealed record ChromiumVersionInfo( + [property: JsonPropertyName("platform")] string Platform, + [property: JsonPropertyName("channel")] string Channel, + [property: JsonPropertyName("version")] string Version); + +internal sealed record ChromiumReference( + [property: JsonPropertyName("url")] string Url, + [property: JsonPropertyName("kind")] string Kind, + [property: JsonPropertyName("label")] string? Label); diff --git a/src/StellaOps.Feedser.Source.Vndr.Chromium/Internal/ChromiumFeedEntry.cs b/src/StellaOps.Feedser.Source.Vndr.Chromium/Internal/ChromiumFeedEntry.cs new file mode 100644 index 00000000..81017793 --- /dev/null +++ b/src/StellaOps.Feedser.Source.Vndr.Chromium/Internal/ChromiumFeedEntry.cs @@ -0,0 +1,24 @@ +namespace StellaOps.Feedser.Source.Vndr.Chromium.Internal; + +public sealed record ChromiumFeedEntry( + string EntryId, + string PostId, + string Title, + Uri DetailUri, + DateTimeOffset Published, + DateTimeOffset? Updated, + string? Summary, + IReadOnlyCollection Categories) +{ + public bool IsSecurityUpdate() + { + if (Categories.Count > 0 && Categories.Contains("Stable updates", StringComparer.OrdinalIgnoreCase)) + { + return true; + } + + return Title.Contains("Stable Channel Update", StringComparison.OrdinalIgnoreCase) + || Title.Contains("Extended Stable", StringComparison.OrdinalIgnoreCase) + || Title.Contains("Stable Channel Desktop", StringComparison.OrdinalIgnoreCase); + } +} diff --git a/src/StellaOps.Feedser.Source.Vndr.Chromium/Internal/ChromiumFeedLoader.cs b/src/StellaOps.Feedser.Source.Vndr.Chromium/Internal/ChromiumFeedLoader.cs new file mode 100644 index 00000000..84888297 --- /dev/null +++ b/src/StellaOps.Feedser.Source.Vndr.Chromium/Internal/ChromiumFeedLoader.cs @@ -0,0 +1,147 @@ +using System.ServiceModel.Syndication; +using System.Xml; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Options; +using StellaOps.Feedser.Source.Vndr.Chromium.Configuration; + +namespace StellaOps.Feedser.Source.Vndr.Chromium.Internal; + +public sealed class ChromiumFeedLoader +{ + private readonly IHttpClientFactory _httpClientFactory; + private readonly ChromiumOptions _options; + private readonly ILogger _logger; + + public ChromiumFeedLoader(IHttpClientFactory httpClientFactory, IOptions options, ILogger logger) + { + _httpClientFactory = httpClientFactory ?? throw new ArgumentNullException(nameof(httpClientFactory)); + _options = options?.Value ?? throw new ArgumentNullException(nameof(options)); + _logger = logger ?? throw new ArgumentNullException(nameof(logger)); + } + + public async Task> LoadAsync(DateTimeOffset windowStart, DateTimeOffset windowEnd, CancellationToken cancellationToken) + { + var client = _httpClientFactory.CreateClient(ChromiumOptions.HttpClientName); + var results = new List(); + var startIndex = 1; + + for (var page = 0; page < _options.MaxFeedPages; page++) + { + var requestUri = BuildRequestUri(startIndex); + using var response = await client.GetAsync(requestUri, cancellationToken).ConfigureAwait(false); + response.EnsureSuccessStatusCode(); + + await using var stream = await response.Content.ReadAsStreamAsync(cancellationToken).ConfigureAwait(false); + using var reader = XmlReader.Create(stream); + var feed = SyndicationFeed.Load(reader); + if (feed is null || feed.Items is null) + { + break; + } + + var pageEntries = new List(); + foreach (var entry in feed.Items) + { + var published = entry.PublishDate != DateTimeOffset.MinValue + ? entry.PublishDate.ToUniversalTime() + : entry.LastUpdatedTime.ToUniversalTime(); + + if (published > windowEnd || published < windowStart - _options.WindowOverlap) + { + continue; + } + + var detailUri = entry.Links.FirstOrDefault(link => string.Equals(link.RelationshipType, "alternate", StringComparison.OrdinalIgnoreCase))?.Uri; + if (detailUri is null) + { + continue; + } + + var postId = ExtractPostId(detailUri); + if (string.IsNullOrEmpty(postId)) + { + continue; + } + + var categories = entry.Categories.Select(static cat => cat.Name).Where(static name => !string.IsNullOrWhiteSpace(name)).ToArray(); + var chromiumEntry = new ChromiumFeedEntry( + entry.Id ?? detailUri.ToString(), + postId, + entry.Title?.Text?.Trim() ?? postId, + detailUri, + published, + entry.LastUpdatedTime == DateTimeOffset.MinValue ? null : entry.LastUpdatedTime.ToUniversalTime(), + entry.Summary?.Text?.Trim(), + categories); + + if (chromiumEntry.Published >= windowStart && chromiumEntry.Published <= windowEnd) + { + pageEntries.Add(chromiumEntry); + } + } + + if (pageEntries.Count == 0) + { + var oldest = feed.Items?.Select(static item => item.PublishDate).Where(static dt => dt != DateTimeOffset.MinValue).OrderBy(static dt => dt).FirstOrDefault(); + if (oldest.HasValue && oldest.Value.ToUniversalTime() < windowStart) + { + break; + } + } + + results.AddRange(pageEntries); + + if (feed.Items?.Any() != true) + { + break; + } + + var nextLink = feed.Links?.FirstOrDefault(link => string.Equals(link.RelationshipType, "next", StringComparison.OrdinalIgnoreCase))?.Uri; + if (nextLink is null) + { + break; + } + + startIndex += _options.MaxEntriesPerPage; + } + + return results + .DistinctBy(static entry => entry.DetailUri) + .OrderBy(static entry => entry.Published) + .ToArray(); + } + + private Uri BuildRequestUri(int startIndex) + { + var builder = new UriBuilder(_options.FeedUri); + var query = new List(); + + if (!string.IsNullOrEmpty(builder.Query)) + { + query.Add(builder.Query.TrimStart('?')); + } + + query.Add($"max-results={_options.MaxEntriesPerPage}"); + query.Add($"start-index={startIndex}"); + query.Add("redirect=false"); + builder.Query = string.Join('&', query); + return builder.Uri; + } + + private static string ExtractPostId(Uri detailUri) + { + var segments = detailUri.Segments; + if (segments.Length == 0) + { + return detailUri.AbsoluteUri; + } + + var last = segments[^1].Trim('/'); + if (last.EndsWith(".html", StringComparison.OrdinalIgnoreCase)) + { + last = last[..^5]; + } + + return last.Replace('/', '-'); + } +} diff --git a/src/StellaOps.Feedser.Source.Vndr.Chromium/Internal/ChromiumMapper.cs b/src/StellaOps.Feedser.Source.Vndr.Chromium/Internal/ChromiumMapper.cs new file mode 100644 index 00000000..a534c38d --- /dev/null +++ b/src/StellaOps.Feedser.Source.Vndr.Chromium/Internal/ChromiumMapper.cs @@ -0,0 +1,174 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Globalization; +using StellaOps.Feedser.Models; +using StellaOps.Feedser.Storage.Mongo.PsirtFlags; + +namespace StellaOps.Feedser.Source.Vndr.Chromium.Internal; + +internal static class ChromiumMapper +{ + private const string VendorIdentifier = "google:chrome"; + + public static (Advisory Advisory, PsirtFlagRecord Flag) Map(ChromiumDto dto, string sourceName, DateTimeOffset recordedAt) + { + ArgumentNullException.ThrowIfNull(dto); + ArgumentException.ThrowIfNullOrEmpty(sourceName); + + var advisoryKey = $"chromium/post/{dto.PostId}"; + var provenance = new AdvisoryProvenance(sourceName, "document", dto.PostId, recordedAt.ToUniversalTime()); + + var aliases = BuildAliases(dto).ToArray(); + var references = BuildReferences(dto, provenance).ToArray(); + var affectedPackages = BuildAffected(dto, provenance).ToArray(); + + var advisory = new Advisory( + advisoryKey, + dto.Title, + dto.Summary, + language: "en", + dto.Published.ToUniversalTime(), + dto.Updated?.ToUniversalTime(), + severity: null, + exploitKnown: false, + aliases, + references, + affectedPackages, + Array.Empty(), + new[] { provenance }); + + var flag = new PsirtFlagRecord( + advisoryKey, + "Google", + sourceName, + dto.PostId, + recordedAt.ToUniversalTime()); + + return (advisory, flag); + } + + private static IEnumerable BuildAliases(ChromiumDto dto) + { + yield return $"CHROMIUM-POST:{dto.PostId}"; + yield return $"CHROMIUM-POST:{dto.Published:yyyy-MM-dd}"; + + foreach (var cve in dto.Cves) + { + yield return cve; + } + } + + private static IEnumerable BuildReferences(ChromiumDto dto, AdvisoryProvenance provenance) + { + var comparer = StringComparer.OrdinalIgnoreCase; + var references = new List<(AdvisoryReference Reference, int Priority)> + { + (new AdvisoryReference(dto.DetailUrl, "advisory", "chromium-blog", summary: null, provenance), 0), + }; + + foreach (var reference in dto.References) + { + var summary = string.IsNullOrWhiteSpace(reference.Label) ? null : reference.Label; + var sourceTag = string.IsNullOrWhiteSpace(reference.Kind) ? null : reference.Kind; + var advisoryReference = new AdvisoryReference(reference.Url, reference.Kind, sourceTag, summary, provenance); + references.Add((advisoryReference, 1)); + } + + return references + .GroupBy(tuple => tuple.Reference.Url, comparer) + .Select(group => group + .OrderBy(t => t.Priority) + .ThenBy(t => t.Reference.Kind ?? string.Empty, comparer) + .ThenBy(t => t.Reference.SourceTag ?? string.Empty, comparer) + .ThenBy(t => t.Reference.Url, comparer) + .First()) + .OrderBy(t => t.Priority) + .ThenBy(t => t.Reference.Kind ?? string.Empty, comparer) + .ThenBy(t => t.Reference.Url, comparer) + .Select(t => t.Reference); + } + + private static IEnumerable BuildAffected(ChromiumDto dto, AdvisoryProvenance provenance) + { + foreach (var version in dto.Versions) + { + var identifier = version.Channel switch + { + "extended-stable" => $"{VendorIdentifier}:extended-stable", + "beta" => $"{VendorIdentifier}:beta", + "dev" => $"{VendorIdentifier}:dev", + _ => VendorIdentifier, + }; + + var range = new AffectedVersionRange( + rangeKind: "vendor", + introducedVersion: null, + fixedVersion: version.Version, + lastAffectedVersion: null, + rangeExpression: null, + provenance, + primitives: BuildRangePrimitives(version)); + + yield return new AffectedPackage( + AffectedPackageTypes.Vendor, + identifier, + version.Platform, + new[] { range }, + statuses: Array.Empty(), + provenance: new[] { provenance }); + } + } + + private static RangePrimitives? BuildRangePrimitives(ChromiumVersionInfo version) + { + var extensions = new Dictionary(StringComparer.Ordinal); + AddExtension(extensions, "chromium.channel", version.Channel); + AddExtension(extensions, "chromium.platform", version.Platform); + AddExtension(extensions, "chromium.version.raw", version.Version); + + if (Version.TryParse(version.Version, out var parsed)) + { + AddExtension(extensions, "chromium.version.normalized", BuildNormalizedVersion(parsed)); + extensions["chromium.version.major"] = parsed.Major.ToString(CultureInfo.InvariantCulture); + extensions["chromium.version.minor"] = parsed.Minor.ToString(CultureInfo.InvariantCulture); + + if (parsed.Build >= 0) + { + extensions["chromium.version.build"] = parsed.Build.ToString(CultureInfo.InvariantCulture); + } + + if (parsed.Revision >= 0) + { + extensions["chromium.version.patch"] = parsed.Revision.ToString(CultureInfo.InvariantCulture); + } + } + + return extensions.Count == 0 ? null : new RangePrimitives(null, null, null, extensions); + } + + private static string BuildNormalizedVersion(Version version) + { + if (version.Build >= 0 && version.Revision >= 0) + { + return $"{version.Major}.{version.Minor}.{version.Build}.{version.Revision}"; + } + + if (version.Build >= 0) + { + return $"{version.Major}.{version.Minor}.{version.Build}"; + } + + return $"{version.Major}.{version.Minor}"; + } + + private static void AddExtension(Dictionary extensions, string key, string? value) + { + if (string.IsNullOrWhiteSpace(value)) + { + return; + } + + extensions[key] = value.Trim(); + } +} diff --git a/src/StellaOps.Feedser.Source.Vndr.Chromium/Internal/ChromiumParser.cs b/src/StellaOps.Feedser.Source.Vndr.Chromium/Internal/ChromiumParser.cs new file mode 100644 index 00000000..cd5f70ec --- /dev/null +++ b/src/StellaOps.Feedser.Source.Vndr.Chromium/Internal/ChromiumParser.cs @@ -0,0 +1,282 @@ +using System.Text.RegularExpressions; +using AngleSharp.Dom; +using AngleSharp.Html.Parser; + +namespace StellaOps.Feedser.Source.Vndr.Chromium.Internal; + +internal static class ChromiumParser +{ + private static readonly HtmlParser HtmlParser = new(); + private static readonly Regex CveRegex = new("CVE-\\d{4}-\\d{4,}", RegexOptions.Compiled | RegexOptions.IgnoreCase); + private static readonly Regex VersionRegex = new("(?\\d+\\.\\d+\\.\\d+\\.\\d+)", RegexOptions.Compiled); + + public static ChromiumDto Parse(string html, ChromiumDocumentMetadata metadata) + { + ArgumentException.ThrowIfNullOrEmpty(html); + ArgumentNullException.ThrowIfNull(metadata); + + var document = HtmlParser.ParseDocument(html); + var body = document.QuerySelector("div.post-body") ?? document.Body; + if (body is null) + { + throw new InvalidOperationException("Chromium post body not found."); + } + + var cves = ExtractCves(body); + var versions = ExtractVersions(body); + var platforms = versions.Select(static v => v.Platform).Distinct(StringComparer.OrdinalIgnoreCase).ToArray(); + var references = ExtractReferences(body, metadata.DetailUrl); + + return ChromiumDto.From(metadata, cves, platforms, versions, references); + } + + private static IReadOnlyList ExtractCves(IElement body) + { + var matches = CveRegex.Matches(body.TextContent ?? string.Empty); + return matches + .Select(static match => match.Value.ToUpperInvariant()) + .Distinct(StringComparer.Ordinal) + .OrderBy(static cve => cve, StringComparer.Ordinal) + .ToArray(); + } + + private static IReadOnlyList ExtractVersions(IElement body) + { + var results = new Dictionary(StringComparer.OrdinalIgnoreCase); + var elements = body.QuerySelectorAll("p,li"); + if (elements.Length == 0) + { + elements = body.QuerySelectorAll("div,span"); + } + + foreach (var element in elements) + { + var text = element.TextContent?.Trim(); + if (string.IsNullOrEmpty(text)) + { + continue; + } + + var channel = DetermineChannel(text); + foreach (Match match in VersionRegex.Matches(text)) + { + var version = match.Groups["version"].Value; + var platform = DeterminePlatform(text, match); + var key = string.Join('|', platform.ToLowerInvariant(), channel.ToLowerInvariant(), version); + if (!results.ContainsKey(key)) + { + results[key] = new ChromiumVersionInfo(platform, channel, version); + } + } + } + + return results.Values + .OrderBy(static v => v.Platform, StringComparer.OrdinalIgnoreCase) + .ThenBy(static v => v.Channel, StringComparer.OrdinalIgnoreCase) + .ThenBy(static v => v.Version, StringComparer.Ordinal) + .ToArray(); + } + + private static string DeterminePlatform(string text, Match match) + { + var after = ExtractSlice(text, match.Index + match.Length, Math.Min(120, text.Length - (match.Index + match.Length))); + var segment = ExtractPlatformSegment(after); + var normalized = NormalizePlatform(segment); + if (!string.IsNullOrEmpty(normalized)) + { + return normalized!; + } + + var before = ExtractSlice(text, Math.Max(0, match.Index - 80), Math.Min(80, match.Index)); + normalized = NormalizePlatform(before + " " + after); + return string.IsNullOrEmpty(normalized) ? "desktop" : normalized!; + } + + private static string DetermineChannel(string text) + { + if (text.Contains("Extended Stable", StringComparison.OrdinalIgnoreCase)) + { + return "extended-stable"; + } + + if (text.Contains("Beta", StringComparison.OrdinalIgnoreCase)) + { + return "beta"; + } + + if (text.Contains("Dev", StringComparison.OrdinalIgnoreCase)) + { + return "dev"; + } + + return "stable"; + } + + private static string ExtractSlice(string text, int start, int length) + { + if (length <= 0) + { + return string.Empty; + } + + return text.Substring(start, length); + } + + private static string ExtractPlatformSegment(string after) + { + if (string.IsNullOrEmpty(after)) + { + return string.Empty; + } + + var forIndex = after.IndexOf("for ", StringComparison.OrdinalIgnoreCase); + if (forIndex < 0) + { + return string.Empty; + } + + var remainder = after[(forIndex + 4)..]; + var terminatorIndex = remainder.IndexOfAny(new[] { '.', ';', '\n', '(', ')' }); + if (terminatorIndex >= 0) + { + remainder = remainder[..terminatorIndex]; + } + + var digitIndex = remainder.IndexOfAny("0123456789".ToCharArray()); + if (digitIndex >= 0) + { + remainder = remainder[..digitIndex]; + } + + var whichIndex = remainder.IndexOf(" which", StringComparison.OrdinalIgnoreCase); + if (whichIndex >= 0) + { + remainder = remainder[..whichIndex]; + } + + return remainder.Trim(); + } + + private static string? NormalizePlatform(string? value) + { + if (string.IsNullOrWhiteSpace(value)) + { + return null; + } + + var normalized = value.Replace("/", " ", StringComparison.OrdinalIgnoreCase) + .Replace(" and ", " ", StringComparison.OrdinalIgnoreCase) + .Replace("&", " ", StringComparison.OrdinalIgnoreCase) + .Trim(); + + if (normalized.Contains("android", StringComparison.OrdinalIgnoreCase)) + { + return "android"; + } + + if (normalized.Contains("chromeos flex", StringComparison.OrdinalIgnoreCase)) + { + return "chromeos-flex"; + } + + if (normalized.Contains("chromeos", StringComparison.OrdinalIgnoreCase) || normalized.Contains("chrome os", StringComparison.OrdinalIgnoreCase)) + { + return "chromeos"; + } + + if (normalized.Contains("linux", StringComparison.OrdinalIgnoreCase)) + { + return "linux"; + } + + var hasWindows = normalized.Contains("windows", StringComparison.OrdinalIgnoreCase); + var hasMac = normalized.Contains("mac", StringComparison.OrdinalIgnoreCase); + + if (hasWindows && hasMac) + { + return "windows-mac"; + } + + if (hasWindows) + { + return "windows"; + } + + if (hasMac) + { + return "mac"; + } + + return null; + } + + private static IReadOnlyList ExtractReferences(IElement body, Uri detailUri) + { + var references = new Dictionary(StringComparer.OrdinalIgnoreCase); + foreach (var anchor in body.QuerySelectorAll("a[href]")) + { + var href = anchor.GetAttribute("href"); + if (string.IsNullOrWhiteSpace(href)) + { + continue; + } + + if (!Uri.TryCreate(href.Trim(), UriKind.Absolute, out var linkUri)) + { + continue; + } + + if (string.Equals(linkUri.AbsoluteUri, detailUri.AbsoluteUri, StringComparison.OrdinalIgnoreCase)) + { + continue; + } + + if (!string.Equals(linkUri.Scheme, Uri.UriSchemeHttp, StringComparison.OrdinalIgnoreCase) + && !string.Equals(linkUri.Scheme, Uri.UriSchemeHttps, StringComparison.OrdinalIgnoreCase)) + { + continue; + } + + var kind = ClassifyReference(linkUri); + var label = anchor.TextContent?.Trim(); + + if (!references.ContainsKey(linkUri.AbsoluteUri)) + { + references[linkUri.AbsoluteUri] = new ChromiumReference(linkUri.AbsoluteUri, kind, string.IsNullOrWhiteSpace(label) ? null : label); + } + } + + return references.Values + .OrderBy(static r => r.Url, StringComparer.Ordinal) + .ThenBy(static r => r.Kind, StringComparer.Ordinal) + .ToArray(); + } + + private static string ClassifyReference(Uri uri) + { + var host = uri.Host; + if (host.Contains("googlesource.com", StringComparison.OrdinalIgnoreCase)) + { + return "changelog"; + } + + if (host.Contains("issues.chromium.org", StringComparison.OrdinalIgnoreCase) + || host.Contains("bugs.chromium.org", StringComparison.OrdinalIgnoreCase) + || host.Contains("crbug.com", StringComparison.OrdinalIgnoreCase)) + { + return "bug"; + } + + if (host.Contains("chromium.org", StringComparison.OrdinalIgnoreCase)) + { + return "doc"; + } + + if (host.Contains("google.com", StringComparison.OrdinalIgnoreCase)) + { + return "google"; + } + + return "reference"; + } +} diff --git a/src/StellaOps.Feedser.Source.Vndr.Chromium/Internal/ChromiumSchemaProvider.cs b/src/StellaOps.Feedser.Source.Vndr.Chromium/Internal/ChromiumSchemaProvider.cs new file mode 100644 index 00000000..33854ffb --- /dev/null +++ b/src/StellaOps.Feedser.Source.Vndr.Chromium/Internal/ChromiumSchemaProvider.cs @@ -0,0 +1,25 @@ +using System.IO; +using System.Reflection; +using System.Threading; +using Json.Schema; + +namespace StellaOps.Feedser.Source.Vndr.Chromium.Internal; + +internal static class ChromiumSchemaProvider +{ + private static readonly Lazy Cached = new(Load, LazyThreadSafetyMode.ExecutionAndPublication); + + public static JsonSchema Schema => Cached.Value; + + private static JsonSchema Load() + { + var assembly = typeof(ChromiumSchemaProvider).GetTypeInfo().Assembly; + const string resourceName = "StellaOps.Feedser.Source.Vndr.Chromium.Schemas.chromium-post.schema.json"; + + using var stream = assembly.GetManifestResourceStream(resourceName) + ?? throw new InvalidOperationException($"Embedded schema '{resourceName}' not found."); + using var reader = new StreamReader(stream); + var schemaText = reader.ReadToEnd(); + return JsonSchema.FromText(schemaText); + } +} diff --git a/src/StellaOps.Feedser.Source.Vndr.Chromium/Properties/AssemblyInfo.cs b/src/StellaOps.Feedser.Source.Vndr.Chromium/Properties/AssemblyInfo.cs new file mode 100644 index 00000000..af25849d --- /dev/null +++ b/src/StellaOps.Feedser.Source.Vndr.Chromium/Properties/AssemblyInfo.cs @@ -0,0 +1,3 @@ +using System.Runtime.CompilerServices; + +[assembly: InternalsVisibleTo("StellaOps.Feedser.Source.Vndr.Chromium.Tests")] diff --git a/src/StellaOps.Feedser.Source.Vndr.Chromium/Schemas/chromium-post.schema.json b/src/StellaOps.Feedser.Source.Vndr.Chromium/Schemas/chromium-post.schema.json new file mode 100644 index 00000000..6f54776d --- /dev/null +++ b/src/StellaOps.Feedser.Source.Vndr.Chromium/Schemas/chromium-post.schema.json @@ -0,0 +1,97 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://stellaops.example/schemas/chromium-post.schema.json", + "type": "object", + "required": [ + "postId", + "title", + "detailUrl", + "published", + "cves", + "platforms", + "versions", + "references" + ], + "properties": { + "postId": { + "type": "string", + "minLength": 1 + }, + "title": { + "type": "string", + "minLength": 1 + }, + "detailUrl": { + "type": "string", + "format": "uri" + }, + "published": { + "type": "string", + "format": "date-time" + }, + "updated": { + "type": ["string", "null"], + "format": "date-time" + }, + "summary": { + "type": ["string", "null"] + }, + "cves": { + "type": "array", + "uniqueItems": true, + "items": { + "type": "string", + "pattern": "^CVE-\\d{4}-\\d{4,}$" + } + }, + "platforms": { + "type": "array", + "items": { + "type": "string", + "minLength": 1 + } + }, + "versions": { + "type": "array", + "minItems": 1, + "items": { + "type": "object", + "required": ["platform", "channel", "version"], + "properties": { + "platform": { + "type": "string", + "minLength": 1 + }, + "channel": { + "type": "string", + "minLength": 1 + }, + "version": { + "type": "string", + "minLength": 4 + } + } + } + }, + "references": { + "type": "array", + "items": { + "type": "object", + "required": ["url", "kind"], + "properties": { + "url": { + "type": "string", + "format": "uri" + }, + "kind": { + "type": "string", + "minLength": 1 + }, + "label": { + "type": ["string", "null"] + } + } + } + } + } +} diff --git a/src/StellaOps.Feedser.Source.Vndr.Chromium/StellaOps.Feedser.Source.Vndr.Chromium.csproj b/src/StellaOps.Feedser.Source.Vndr.Chromium/StellaOps.Feedser.Source.Vndr.Chromium.csproj new file mode 100644 index 00000000..31406c32 --- /dev/null +++ b/src/StellaOps.Feedser.Source.Vndr.Chromium/StellaOps.Feedser.Source.Vndr.Chromium.csproj @@ -0,0 +1,32 @@ + + + + net10.0 + enable + enable + + + + + + + + + + + + + + + + + + + + + + <_Parameter1>StellaOps.Feedser.Source.Vndr.Chromium.Tests + + + + diff --git a/src/StellaOps.Feedser.Source.Vndr.Chromium/TASKS.md b/src/StellaOps.Feedser.Source.Vndr.Chromium/TASKS.md new file mode 100644 index 00000000..70a5cf6c --- /dev/null +++ b/src/StellaOps.Feedser.Source.Vndr.Chromium/TASKS.md @@ -0,0 +1,17 @@ +# Source.Vndr.Chromium — Task Board + +| ID | Task | Owner | Status | Depends On | Notes | +|------|-----------------------------------------------|-------|--------|------------|-------| +| CH1 | Blog crawl + cursor | Conn | DONE | Common | Sliding window feed reader with cursor persisted. | +| CH2 | Post parser → DTO (CVEs, versions, refs) | QA | DONE | | AngleSharp parser normalizes CVEs, versions, references. | +| CH3 | Canonical mapping (aliases/refs/affected-hint)| Conn | DONE | Models | Deterministic advisory mapping with psirt flags. | +| CH4 | Snapshot tests + resume | QA | DONE | Storage | Deterministic snapshot plus resume scenario via Mongo state. | +| CH5 | Observability | QA | DONE | | Metered fetch/parse/map counters. | +| CH6 | SourceState + SHA dedupe | Conn | DONE | Storage | Cursor tracks SHA cache to skip unchanged posts. | +| CH7 | Stabilize resume integration (preserve pending docs across provider instances) | QA | DONE | Storage.Mongo | Resume integration test exercises pending docs across providers via shared Mongo. | +| CH8 | Mark failed parse documents | Conn | DONE | Storage.Mongo | Parse pipeline marks failures; unit tests assert status transitions. | +| CH9 | Reference dedupe & ordering | Conn | DONE | Models | Mapper groups references by URL and sorts deterministically. | +| CH10 | Range primitives + provenance instrumentation | Conn | DONE | Models, Storage.Mongo | Vendor primitives + logging in place, resume metrics updated, snapshots refreshed. | + +## Changelog +- YYYY-MM-DD: Created. diff --git a/src/StellaOps.Feedser.Source.Vndr.Cisco/AGENTS.md b/src/StellaOps.Feedser.Source.Vndr.Cisco/AGENTS.md new file mode 100644 index 00000000..08334c1d --- /dev/null +++ b/src/StellaOps.Feedser.Source.Vndr.Cisco/AGENTS.md @@ -0,0 +1,30 @@ +# AGENTS +## Role +Implement the Cisco security advisory connector to ingest Cisco PSIRT bulletins for Feedser. + +## Scope +- Identify Cisco advisory feeds/APIs (XML, HTML, JSON) and define incremental fetch strategy. +- Implement fetch/cursor pipeline with retry/backoff and document dedupe. +- Parse advisories to extract summary, affected products, Cisco bug IDs, CVEs, mitigation guidance. +- Map advisories into canonical `Advisory` records with aliases, references, affected packages, and range primitives (e.g., SemVer/IOS version metadata). +- Provide deterministic fixtures and regression tests. + +## Participants +- `Source.Common`, `Storage.Mongo`, `Feedser.Models`, `Feedser.Testing`. + +## Interfaces & Contracts +- Job kinds: `cisco:fetch`, `cisco:parse`, `cisco:map`. +- Persist upstream metadata (e.g., `Last-Modified`, `advisoryId`). +- Alias set should include Cisco advisory IDs, bug IDs, and CVEs. + +## In/Out of scope +In scope: Cisco PSIRT advisories, range primitive coverage. +Out of scope: Non-security Cisco release notes. + +## Observability & Security Expectations +- Log fetch/mapping statistics, respect Cisco API rate limits, sanitise HTML. +- Handle authentication tokens if API requires them. + +## Tests +- Add `StellaOps.Feedser.Source.Vndr.Cisco.Tests` with canned fixtures for fetch/parse/map. +- Snapshot canonical advisories and support fixture regeneration. diff --git a/src/StellaOps.Feedser.Source.Vndr.Cisco/Class1.cs b/src/StellaOps.Feedser.Source.Vndr.Cisco/Class1.cs new file mode 100644 index 00000000..89524ed7 --- /dev/null +++ b/src/StellaOps.Feedser.Source.Vndr.Cisco/Class1.cs @@ -0,0 +1,29 @@ +using System; +using System.Threading; +using System.Threading.Tasks; +using StellaOps.Plugin; + +namespace StellaOps.Feedser.Source.Vndr.Cisco; + +public sealed class VndrCiscoConnectorPlugin : IConnectorPlugin +{ + public string Name => "vndr-cisco"; + + public bool IsAvailable(IServiceProvider services) => true; + + public IFeedConnector Create(IServiceProvider services) => new StubConnector(Name); + + private sealed class StubConnector : IFeedConnector + { + public StubConnector(string sourceName) => SourceName = sourceName; + + public string SourceName { get; } + + public Task FetchAsync(IServiceProvider services, CancellationToken cancellationToken) => Task.CompletedTask; + + public Task ParseAsync(IServiceProvider services, CancellationToken cancellationToken) => Task.CompletedTask; + + public Task MapAsync(IServiceProvider services, CancellationToken cancellationToken) => Task.CompletedTask; + } +} + diff --git a/src/StellaOps.Feedser.Source.Vndr.Cisco/StellaOps.Feedser.Source.Vndr.Cisco.csproj b/src/StellaOps.Feedser.Source.Vndr.Cisco/StellaOps.Feedser.Source.Vndr.Cisco.csproj new file mode 100644 index 00000000..f7f2c154 --- /dev/null +++ b/src/StellaOps.Feedser.Source.Vndr.Cisco/StellaOps.Feedser.Source.Vndr.Cisco.csproj @@ -0,0 +1,16 @@ + + + + net10.0 + enable + enable + + + + + + + + + + diff --git a/src/StellaOps.Feedser.Source.Vndr.Cisco/TASKS.md b/src/StellaOps.Feedser.Source.Vndr.Cisco/TASKS.md new file mode 100644 index 00000000..7e06e703 --- /dev/null +++ b/src/StellaOps.Feedser.Source.Vndr.Cisco/TASKS.md @@ -0,0 +1,10 @@ +# TASKS +| Task | Owner(s) | Depends on | Notes | +|---|---|---|---| +|FEEDCONN-CISCO-02-001 Confirm Cisco PSIRT data source|BE-Conn-Cisco|Research|**DOING** – 2025-10-11: Logged public PSIRT RSS and openVuln Security Advisories API docs; need to confirm API key workflow, pagination, throttling, and select primary ingestion path.| +|FEEDCONN-CISCO-02-002 Fetch pipeline & state persistence|BE-Conn-Cisco|Source.Common, Storage.Mongo|**TODO** – Implement fetch job with cursor/backoff, persist raw documents, dedupe repeated advisories.| +|FEEDCONN-CISCO-02-003 Parser & DTO implementation|BE-Conn-Cisco|Source.Common|**TODO** – Parse advisories, extract Cisco bug IDs, product families, version ranges, CVEs, references.| +|FEEDCONN-CISCO-02-004 Canonical mapping & range primitives|BE-Conn-Cisco|Models|**TODO** – Map advisories into canonical records with aliases, references, range primitives (SemVer/IOS/ASA versions). Sync scheme decisions and deadlines via `../StellaOps.Feedser.Merge/RANGE_PRIMITIVES_COORDINATION.md`.| +|FEEDCONN-CISCO-02-005 Deterministic fixtures & tests|QA|Testing|**TODO** – Add fetch/parse/map regression tests; support `UPDATE_CISCO_FIXTURES=1`.| +|FEEDCONN-CISCO-02-006 Telemetry & documentation|DevEx|Docs|**TODO** – Add logging/metrics, document connector usage, update backlog when ready.| +|FEEDCONN-CISCO-02-007 API selection decision memo|BE-Conn-Cisco|Research|**TODO** – Produce recommendation on RSS vs openVuln (scope, auth, SLAs) and share with Merge/Storage for cursor alignment.| diff --git a/src/StellaOps.Feedser.Source.Vndr.Msrc/AGENTS.md b/src/StellaOps.Feedser.Source.Vndr.Msrc/AGENTS.md new file mode 100644 index 00000000..289819ad --- /dev/null +++ b/src/StellaOps.Feedser.Source.Vndr.Msrc/AGENTS.md @@ -0,0 +1,30 @@ +# AGENTS +## Role +Implement the Microsoft Security Response Center (MSRC) connector to ingest Microsoft security updates (Security Updates API / CVRF). + +## Scope +- Identify MSRC data sources (Security Update Guide API, CVRF downloads) and incremental update strategy. +- Implement fetch/cursor pipeline with retry/backoff, handling API keys if required. +- Parse advisories to extract summary, affected products, KBs, CVEs, severities, mitigations. +- Map entries into canonical `Advisory` objects with aliases, references, affected packages, and range primitives (e.g., Windows build numbers, SemVer). +- Provide deterministic fixtures and regression tests. + +## Participants +- `Source.Common`, `Storage.Mongo`, `Feedser.Models`, `Feedser.Testing`. + +## Interfaces & Contracts +- Job kinds: `msrc:fetch`, `msrc:parse`, `msrc:map`. +- Persist upstream metadata (e.g., `lastModified`, `releaseDate`). +- Alias set should include MSRC ID, CVEs, and KB identifiers. + +## In/Out of scope +In scope: Microsoft Security Update Guide advisories. +Out of scope: Non-security Microsoft release notes. + +## Observability & Security Expectations +- Log fetch/mapping stats, respect API rate limits, handle authentication securely. +- Sanitize payloads; validate JSON/CVRF before persistence. + +## Tests +- Add `StellaOps.Feedser.Source.Vndr.Msrc.Tests` with fixtures covering fetch/parse/map. +- Snapshot canonical advisories; support fixture regeneration. diff --git a/src/StellaOps.Feedser.Source.Vndr.Msrc/Class1.cs b/src/StellaOps.Feedser.Source.Vndr.Msrc/Class1.cs new file mode 100644 index 00000000..efee818a --- /dev/null +++ b/src/StellaOps.Feedser.Source.Vndr.Msrc/Class1.cs @@ -0,0 +1,29 @@ +using System; +using System.Threading; +using System.Threading.Tasks; +using StellaOps.Plugin; + +namespace StellaOps.Feedser.Source.Vndr.Msrc; + +public sealed class VndrMsrcConnectorPlugin : IConnectorPlugin +{ + public string Name => "vndr-msrc"; + + public bool IsAvailable(IServiceProvider services) => true; + + public IFeedConnector Create(IServiceProvider services) => new StubConnector(Name); + + private sealed class StubConnector : IFeedConnector + { + public StubConnector(string sourceName) => SourceName = sourceName; + + public string SourceName { get; } + + public Task FetchAsync(IServiceProvider services, CancellationToken cancellationToken) => Task.CompletedTask; + + public Task ParseAsync(IServiceProvider services, CancellationToken cancellationToken) => Task.CompletedTask; + + public Task MapAsync(IServiceProvider services, CancellationToken cancellationToken) => Task.CompletedTask; + } +} + diff --git a/src/StellaOps.Feedser.Source.Vndr.Msrc/StellaOps.Feedser.Source.Vndr.Msrc.csproj b/src/StellaOps.Feedser.Source.Vndr.Msrc/StellaOps.Feedser.Source.Vndr.Msrc.csproj new file mode 100644 index 00000000..f7f2c154 --- /dev/null +++ b/src/StellaOps.Feedser.Source.Vndr.Msrc/StellaOps.Feedser.Source.Vndr.Msrc.csproj @@ -0,0 +1,16 @@ + + + + net10.0 + enable + enable + + + + + + + + + + diff --git a/src/StellaOps.Feedser.Source.Vndr.Msrc/TASKS.md b/src/StellaOps.Feedser.Source.Vndr.Msrc/TASKS.md new file mode 100644 index 00000000..e10e00b3 --- /dev/null +++ b/src/StellaOps.Feedser.Source.Vndr.Msrc/TASKS.md @@ -0,0 +1,10 @@ +# TASKS +| Task | Owner(s) | Depends on | Notes | +|---|---|---|---| +|FEEDCONN-MSRC-02-001 Document MSRC Security Update Guide API|BE-Conn-MSRC|Research|**DOING** – 2025-10-11: Reviewed MSRC Security Update Guide API docs (REST/Graph + CVRF) and open-source client; still need to capture auth token handling, throttling, and delta parameters.| +|FEEDCONN-MSRC-02-002 Fetch pipeline & source state|BE-Conn-MSRC|Source.Common, Storage.Mongo|**TODO** – Implement fetch job with retry/backoff, persist raw documents, manage cursors.| +|FEEDCONN-MSRC-02-003 Parser & DTO implementation|BE-Conn-MSRC|Source.Common|**TODO** – Build DTOs for MSRC advisories (title, description, KB IDs, CVEs, product tree, severity).| +|FEEDCONN-MSRC-02-004 Canonical mapping & range primitives|BE-Conn-MSRC|Models|**TODO** – Map advisories to canonical records with aliases, references, range primitives for product/build coverage. Coordinate scheme naming and normalized outputs with `../StellaOps.Feedser.Merge/RANGE_PRIMITIVES_COORDINATION.md`.| +|FEEDCONN-MSRC-02-005 Deterministic fixtures/tests|QA|Testing|**TODO** – Add regression tests with fixtures; support `UPDATE_MSRC_FIXTURES=1`.| +|FEEDCONN-MSRC-02-006 Telemetry & documentation|DevEx|Docs|**TODO** – Add logging/metrics and documentation; update backlog once connector is production-ready.| +|FEEDCONN-MSRC-02-007 API contract comparison memo|BE-Conn-MSRC|Research|**TODO** – Summarise REST vs CVRF payload differences, rate limits, and token requirements; recommend primary ingestion path for Feedser.| diff --git a/src/StellaOps.Feedser.Source.Vndr.Oracle.Tests/Oracle/Fixtures/oracle-advisories.snapshot.json b/src/StellaOps.Feedser.Source.Vndr.Oracle.Tests/Oracle/Fixtures/oracle-advisories.snapshot.json new file mode 100644 index 00000000..7c264393 --- /dev/null +++ b/src/StellaOps.Feedser.Source.Vndr.Oracle.Tests/Oracle/Fixtures/oracle-advisories.snapshot.json @@ -0,0 +1,495 @@ +[ + { + "advisoryKey": "oracle/cpuapr2024-01-html", + "affectedPackages": [ + { + "identifier": "Oracle GraalVM for JDK::Libraries", + "platform": "Libraries", + "provenance": [ + { + "fieldMask": [], + "kind": "affected", + "recordedAt": "2024-04-18T00:01:00+00:00", + "source": "vndr-oracle", + "value": "Oracle GraalVM for JDK::Libraries" + } + ], + "statuses": [], + "type": "vendor", + "versionRanges": [ + { + "fixedVersion": null, + "introducedVersion": null, + "lastAffectedVersion": null, + "primitives": { + "evr": null, + "hasVendorExtensions": true, + "nevra": null, + "semVer": null, + "vendorExtensions": { + "oracle.product": "Oracle GraalVM for JDK", + "oracle.productRaw": "Oracle Java SE, Oracle GraalVM for JDK", + "oracle.component": "Libraries", + "oracle.componentRaw": "Libraries", + "oracle.segmentVersions": "21.3.8, 22.0.0", + "oracle.supportedVersions": "Oracle Java SE: 8u401, 11.0.22; Oracle GraalVM for JDK: 21.3.8, 22.0.0", + "oracle.rangeExpression": "21.3.8, 22.0.0 (notes: See Note A for mitigation)", + "oracle.baseExpression": "21.3.8, 22.0.0", + "oracle.notes": "See Note A for mitigation", + "oracle.versionTokens": "21.3.8|22.0.0", + "oracle.versionTokens.normalized": "21.3.8|22.0.0" + } + }, + "provenance": { + "fieldMask": [], + "kind": "range", + "recordedAt": "2024-04-18T00:01:00+00:00", + "source": "vndr-oracle", + "value": "Oracle GraalVM for JDK::Libraries" + }, + "rangeExpression": "21.3.8, 22.0.0 (notes: See Note A for mitigation)", + "rangeKind": "vendor" + } + ] + }, + { + "identifier": "Oracle Java SE::Hotspot", + "platform": "Hotspot", + "provenance": [ + { + "fieldMask": [], + "kind": "affected", + "recordedAt": "2024-04-18T00:01:00+00:00", + "source": "vndr-oracle", + "value": "Oracle Java SE::Hotspot" + } + ], + "statuses": [], + "type": "vendor", + "versionRanges": [ + { + "fixedVersion": "8u401", + "introducedVersion": null, + "lastAffectedVersion": null, + "primitives": { + "evr": null, + "hasVendorExtensions": true, + "nevra": null, + "semVer": null, + "vendorExtensions": { + "oracle.product": "Oracle Java SE", + "oracle.productRaw": "Oracle Java SE", + "oracle.component": "Hotspot", + "oracle.componentRaw": "Hotspot", + "oracle.segmentVersions": "Oracle Java SE: 8u401, 11.0.22", + "oracle.supportedVersions": "Oracle Java SE: 8u401, 11.0.22", + "oracle.rangeExpression": "Oracle Java SE: 8u401, 11.0.22 (notes: Fixed in 8u401 Patch 123456)", + "oracle.baseExpression": "Oracle Java SE: 8u401, 11.0.22", + "oracle.notes": "Fixed in 8u401 Patch 123456", + "oracle.fixedVersion": "8u401", + "oracle.patchNumber": "123456", + "oracle.versionTokens": "Oracle Java SE: 8u401|11.0.22", + "oracle.versionTokens.normalized": "11.0.22" + } + }, + "provenance": { + "fieldMask": [], + "kind": "range", + "recordedAt": "2024-04-18T00:01:00+00:00", + "source": "vndr-oracle", + "value": "Oracle Java SE::Hotspot" + }, + "rangeExpression": "Oracle Java SE: 8u401, 11.0.22 (notes: Fixed in 8u401 Patch 123456)", + "rangeKind": "vendor" + } + ] + }, + { + "identifier": "Oracle Java SE::Libraries", + "platform": "Libraries", + "provenance": [ + { + "fieldMask": [], + "kind": "affected", + "recordedAt": "2024-04-18T00:01:00+00:00", + "source": "vndr-oracle", + "value": "Oracle Java SE::Libraries" + } + ], + "statuses": [], + "type": "vendor", + "versionRanges": [ + { + "fixedVersion": null, + "introducedVersion": null, + "lastAffectedVersion": null, + "primitives": { + "evr": null, + "hasVendorExtensions": true, + "nevra": null, + "semVer": null, + "vendorExtensions": { + "oracle.product": "Oracle Java SE", + "oracle.productRaw": "Oracle Java SE, Oracle GraalVM for JDK", + "oracle.component": "Libraries", + "oracle.componentRaw": "Libraries", + "oracle.segmentVersions": "8u401, 11.0.22", + "oracle.supportedVersions": "Oracle Java SE: 8u401, 11.0.22; Oracle GraalVM for JDK: 21.3.8, 22.0.0", + "oracle.rangeExpression": "8u401, 11.0.22 (notes: See Note A for mitigation)", + "oracle.baseExpression": "8u401, 11.0.22", + "oracle.notes": "See Note A for mitigation", + "oracle.versionTokens": "8u401|11.0.22", + "oracle.versionTokens.normalized": "11.0.22" + } + }, + "provenance": { + "fieldMask": [], + "kind": "range", + "recordedAt": "2024-04-18T00:01:00+00:00", + "source": "vndr-oracle", + "value": "Oracle Java SE::Libraries" + }, + "rangeExpression": "8u401, 11.0.22 (notes: See Note A for mitigation)", + "rangeKind": "vendor" + } + ] + } + ], + "aliases": [ + "CVE-2024-9000", + "CVE-2024-9001", + "ORACLE:CPUAPR2024-01-HTML" + ], + "cvssMetrics": [], + "exploitKnown": false, + "language": "en", + "modified": null, + "provenance": [ + { + "fieldMask": [], + "kind": "document", + "recordedAt": "2024-04-18T00:00:00+00:00", + "source": "vndr-oracle", + "value": "https://www.oracle.com/security-alerts/cpuapr2024-01.html" + }, + { + "fieldMask": [], + "kind": "mapping", + "recordedAt": "2024-04-18T00:01:00+00:00", + "source": "vndr-oracle", + "value": "cpuapr2024-01-html" + } + ], + "published": "2024-04-18T12:30:00+00:00", + "references": [ + { + "kind": "reference", + "provenance": { + "fieldMask": [], + "kind": "reference", + "recordedAt": "2024-04-18T00:01:00+00:00", + "source": "vndr-oracle", + "value": "https://support.oracle.com/kb/123456" + }, + "sourceTag": null, + "summary": null, + "url": "https://support.oracle.com/kb/123456" + }, + { + "kind": "patch", + "provenance": { + "fieldMask": [], + "kind": "reference", + "recordedAt": "2024-04-18T00:01:00+00:00", + "source": "vndr-oracle", + "value": "https://support.oracle.com/rs?type=doc&id=3010001.1" + }, + "sourceTag": "oracle", + "summary": "Oracle Java SE", + "url": "https://support.oracle.com/rs?type=doc&id=3010001.1" + }, + { + "kind": "patch", + "provenance": { + "fieldMask": [], + "kind": "reference", + "recordedAt": "2024-04-18T00:01:00+00:00", + "source": "vndr-oracle", + "value": "https://support.oracle.com/rs?type=doc&id=3010002.1" + }, + "sourceTag": "oracle", + "summary": "Oracle GraalVM", + "url": "https://support.oracle.com/rs?type=doc&id=3010002.1" + }, + { + "kind": "reference", + "provenance": { + "fieldMask": [], + "kind": "reference", + "recordedAt": "2024-04-18T00:01:00+00:00", + "source": "vndr-oracle", + "value": "https://updates.oracle.com/patches/fullpatch" + }, + "sourceTag": null, + "summary": null, + "url": "https://updates.oracle.com/patches/fullpatch" + }, + { + "kind": "advisory", + "provenance": { + "fieldMask": [], + "kind": "reference", + "recordedAt": "2024-04-18T00:01:00+00:00", + "source": "vndr-oracle", + "value": "https://www.cve.org/CVERecord?id=CVE-2024-9000" + }, + "sourceTag": "CVE-2024-9000", + "summary": null, + "url": "https://www.cve.org/CVERecord?id=CVE-2024-9000" + }, + { + "kind": "advisory", + "provenance": { + "fieldMask": [], + "kind": "reference", + "recordedAt": "2024-04-18T00:01:00+00:00", + "source": "vndr-oracle", + "value": "https://www.cve.org/CVERecord?id=CVE-2024-9001" + }, + "sourceTag": "CVE-2024-9001", + "summary": null, + "url": "https://www.cve.org/CVERecord?id=CVE-2024-9001" + }, + { + "kind": "advisory", + "provenance": { + "fieldMask": [], + "kind": "reference", + "recordedAt": "2024-04-18T00:01:00+00:00", + "source": "vndr-oracle", + "value": "https://www.oracle.com/security-alerts/cpuapr2024-01.html" + }, + "sourceTag": "oracle", + "summary": "cpuapr2024 01 html", + "url": "https://www.oracle.com/security-alerts/cpuapr2024-01.html" + } + ], + "severity": null, + "summary": "Oracle CPU April 2024 Advisory 1 Oracle Critical Patch Update Advisory - April 2024 (CPU01) This advisory addresses vulnerabilities in Oracle Java SE and Oracle GraalVM for JDK. It references CVE-2024-9000 and CVE-2024-9001 with additional remediation steps. Affected Products and Versions Patch Availability Document Oracle Java SE, versions 8u401, 11.0.22 Oracle Java SE Oracle GraalVM for JDK, versions 21.3.8, 22.0.0 Oracle GraalVM CVE ID Product Component Protocol Remote Exploit without Auth.? Base Score Attack Vector Attack Complex Privs Req'd User Interact Scope Confidentiality Integrity Availability Supported Versions Affected Notes CVE-2024-9000 Oracle Java SE Hotspot Multiple Yes 9.8 Network Low None Required Changed High High High Oracle Java SE: 8u401, 11.0.22 Fixed in 8u401 Patch 123456 CVE-2024-9001 Oracle Java SE, Oracle GraalVM for JDK Libraries Multiple Yes 7.5 Network High None Required Changed Medium Medium Medium Oracle Java SE: 8u401, 11.0.22; Oracle GraalVM for JDK: 21.3.8, 22.0.0 See Note A for mitigation Note A: Apply interim update 22.0.0.1 for GraalVM. Patch download Support article", + "title": "cpuapr2024 01 html" + }, + { + "advisoryKey": "oracle/cpuapr2024-02-html", + "affectedPackages": [ + { + "identifier": "Oracle Database Server::SQL*Plus", + "platform": "SQL*Plus", + "provenance": [ + { + "fieldMask": [], + "kind": "affected", + "recordedAt": "2024-04-18T00:01:00+00:00", + "source": "vndr-oracle", + "value": "Oracle Database Server::SQL*Plus" + } + ], + "statuses": [], + "type": "vendor", + "versionRanges": [ + { + "fixedVersion": null, + "introducedVersion": null, + "lastAffectedVersion": null, + "primitives": { + "evr": null, + "hasVendorExtensions": true, + "nevra": null, + "semVer": null, + "vendorExtensions": { + "oracle.product": "Oracle Database Server", + "oracle.productRaw": "Oracle Database Server", + "oracle.component": "SQL*Plus", + "oracle.componentRaw": "SQL*Plus", + "oracle.segmentVersions": "Oracle Database Server: 19c, 21c", + "oracle.supportedVersions": "Oracle Database Server: 19c, 21c", + "oracle.rangeExpression": "Oracle Database Server: 19c, 21c (notes: See Note B)", + "oracle.baseExpression": "Oracle Database Server: 19c, 21c", + "oracle.notes": "See Note B", + "oracle.versionTokens": "Oracle Database Server: 19c|21c" + } + }, + "provenance": { + "fieldMask": [], + "kind": "range", + "recordedAt": "2024-04-18T00:01:00+00:00", + "source": "vndr-oracle", + "value": "Oracle Database Server::SQL*Plus" + }, + "rangeExpression": "Oracle Database Server: 19c, 21c (notes: See Note B)", + "rangeKind": "vendor" + } + ] + }, + { + "identifier": "Oracle WebLogic Server::Console", + "platform": "Console", + "provenance": [ + { + "fieldMask": [], + "kind": "affected", + "recordedAt": "2024-04-18T00:01:00+00:00", + "source": "vndr-oracle", + "value": "Oracle WebLogic Server::Console" + } + ], + "statuses": [], + "type": "vendor", + "versionRanges": [ + { + "fixedVersion": "99999999", + "introducedVersion": null, + "lastAffectedVersion": null, + "primitives": { + "evr": null, + "hasVendorExtensions": true, + "nevra": null, + "semVer": null, + "vendorExtensions": { + "oracle.product": "Oracle WebLogic Server", + "oracle.productRaw": "Oracle WebLogic Server", + "oracle.component": "Console", + "oracle.componentRaw": "Console", + "oracle.segmentVersions": "Oracle WebLogic Server: 14.1.1.0.0", + "oracle.supportedVersions": "Oracle WebLogic Server: 14.1.1.0.0", + "oracle.rangeExpression": "Oracle WebLogic Server: 14.1.1.0.0 (notes: Patch 99999999 available)", + "oracle.baseExpression": "Oracle WebLogic Server: 14.1.1.0.0", + "oracle.notes": "Patch 99999999 available", + "oracle.fixedVersion": "99999999", + "oracle.patchNumber": "99999999", + "oracle.versionTokens": "Oracle WebLogic Server: 14.1.1.0.0" + } + }, + "provenance": { + "fieldMask": [], + "kind": "range", + "recordedAt": "2024-04-18T00:01:00+00:00", + "source": "vndr-oracle", + "value": "Oracle WebLogic Server::Console" + }, + "rangeExpression": "Oracle WebLogic Server: 14.1.1.0.0 (notes: Patch 99999999 available)", + "rangeKind": "vendor" + } + ] + } + ], + "aliases": [ + "CVE-2024-9100", + "CVE-2024-9101", + "ORACLE:CPUAPR2024-02-HTML" + ], + "cvssMetrics": [], + "exploitKnown": false, + "language": "en", + "modified": null, + "provenance": [ + { + "fieldMask": [], + "kind": "document", + "recordedAt": "2024-04-18T00:00:00+00:00", + "source": "vndr-oracle", + "value": "https://www.oracle.com/security-alerts/cpuapr2024-02.html" + }, + { + "fieldMask": [], + "kind": "mapping", + "recordedAt": "2024-04-18T00:01:00+00:00", + "source": "vndr-oracle", + "value": "cpuapr2024-02-html" + } + ], + "published": "2024-04-19T08:15:00+00:00", + "references": [ + { + "kind": "reference", + "provenance": { + "fieldMask": [], + "kind": "reference", + "recordedAt": "2024-04-18T00:01:00+00:00", + "source": "vndr-oracle", + "value": "https://support.oracle.com/kb/789012" + }, + "sourceTag": null, + "summary": null, + "url": "https://support.oracle.com/kb/789012" + }, + { + "kind": "patch", + "provenance": { + "fieldMask": [], + "kind": "reference", + "recordedAt": "2024-04-18T00:01:00+00:00", + "source": "vndr-oracle", + "value": "https://support.oracle.com/rs?type=doc&id=3010100.1" + }, + "sourceTag": "oracle", + "summary": "Fusion Middleware", + "url": "https://support.oracle.com/rs?type=doc&id=3010100.1" + }, + { + "kind": "patch", + "provenance": { + "fieldMask": [], + "kind": "reference", + "recordedAt": "2024-04-18T00:01:00+00:00", + "source": "vndr-oracle", + "value": "https://support.oracle.com/rs?type=doc&id=3010101.1" + }, + "sourceTag": "oracle", + "summary": "Database", + "url": "https://support.oracle.com/rs?type=doc&id=3010101.1" + }, + { + "kind": "advisory", + "provenance": { + "fieldMask": [], + "kind": "reference", + "recordedAt": "2024-04-18T00:01:00+00:00", + "source": "vndr-oracle", + "value": "https://www.cve.org/CVERecord?id=CVE-2024-9100" + }, + "sourceTag": "CVE-2024-9100", + "summary": null, + "url": "https://www.cve.org/CVERecord?id=CVE-2024-9100" + }, + { + "kind": "advisory", + "provenance": { + "fieldMask": [], + "kind": "reference", + "recordedAt": "2024-04-18T00:01:00+00:00", + "source": "vndr-oracle", + "value": "https://www.cve.org/CVERecord?id=CVE-2024-9101" + }, + "sourceTag": "CVE-2024-9101", + "summary": null, + "url": "https://www.cve.org/CVERecord?id=CVE-2024-9101" + }, + { + "kind": "advisory", + "provenance": { + "fieldMask": [], + "kind": "reference", + "recordedAt": "2024-04-18T00:01:00+00:00", + "source": "vndr-oracle", + "value": "https://www.oracle.com/security-alerts/cpuapr2024-02.html" + }, + "sourceTag": "oracle", + "summary": "cpuapr2024 02 html", + "url": "https://www.oracle.com/security-alerts/cpuapr2024-02.html" + } + ], + "severity": null, + "summary": "Oracle CPU April 2024 Advisory 2 Oracle Security Alert Advisory - April 2024 (CPU02) Mitigations for Oracle WebLogic Server and Oracle Database Server. Includes references to CVE-2024-9100 with additional product components. Affected Products and Versions Patch Availability Document Oracle WebLogic Server, versions 14.1.1.0.0 Fusion Middleware Oracle Database Server, versions 19c, 21c Database CVE ID Product Component Protocol Remote Exploit without Auth.? Base Score Attack Vector Attack Complex Privs Req'd User Interact Scope Confidentiality Integrity Availability Supported Versions Affected Notes CVE-2024-9100 Oracle WebLogic Server Console HTTP Yes 8.1 Network Low Low Required Changed High High High Oracle WebLogic Server: 14.1.1.0.0 Patch 99999999 available CVE-2024-9101 Oracle Database Server SQL*Plus Multiple No 5.4 Local Low Low None Unchanged Medium Low Low Oracle Database Server: 19c, 21c See Note B Note B: Customers should review Support Doc 3010101.1 for mitigation guidance. More details at Support KB .", + "title": "cpuapr2024 02 html" + } +] \ No newline at end of file diff --git a/src/StellaOps.Feedser.Source.Vndr.Oracle.Tests/Oracle/Fixtures/oracle-calendar-cpuapr2024-single.html b/src/StellaOps.Feedser.Source.Vndr.Oracle.Tests/Oracle/Fixtures/oracle-calendar-cpuapr2024-single.html new file mode 100644 index 00000000..f52f4abe --- /dev/null +++ b/src/StellaOps.Feedser.Source.Vndr.Oracle.Tests/Oracle/Fixtures/oracle-calendar-cpuapr2024-single.html @@ -0,0 +1,7 @@ + + + + + diff --git a/src/StellaOps.Feedser.Source.Vndr.Oracle.Tests/Oracle/Fixtures/oracle-calendar-cpuapr2024.html b/src/StellaOps.Feedser.Source.Vndr.Oracle.Tests/Oracle/Fixtures/oracle-calendar-cpuapr2024.html new file mode 100644 index 00000000..4a166b6f --- /dev/null +++ b/src/StellaOps.Feedser.Source.Vndr.Oracle.Tests/Oracle/Fixtures/oracle-calendar-cpuapr2024.html @@ -0,0 +1,8 @@ + + + + + diff --git a/src/StellaOps.Feedser.Source.Vndr.Oracle.Tests/Oracle/Fixtures/oracle-detail-cpuapr2024-01.html b/src/StellaOps.Feedser.Source.Vndr.Oracle.Tests/Oracle/Fixtures/oracle-detail-cpuapr2024-01.html new file mode 100644 index 00000000..fbf0b80f --- /dev/null +++ b/src/StellaOps.Feedser.Source.Vndr.Oracle.Tests/Oracle/Fixtures/oracle-detail-cpuapr2024-01.html @@ -0,0 +1,108 @@ + + + Oracle CPU April 2024 Advisory 1 + + + +

      Oracle Critical Patch Update Advisory - April 2024 (CPU01)

      +

      + This advisory addresses vulnerabilities in Oracle Java SE and Oracle GraalVM for JDK. + It references CVE-2024-9000 and CVE-2024-9001 with additional remediation steps. +

      + +
      +
      + + + + + + + + + + + + + + + + + +
      Affected Products and VersionsPatch Availability Document
      Oracle Java SE, versions 8u401, 11.0.22Oracle Java SE
      Oracle GraalVM for JDK, versions 21.3.8, 22.0.0Oracle GraalVM
      +
      +
      + +
      +
      + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
      CVE IDProductComponentProtocolRemote Exploit without Auth.?Base ScoreAttack VectorAttack ComplexPrivs Req'dUser InteractScopeConfidentialityIntegrityAvailabilitySupported Versions AffectedNotes
      CVE-2024-9000Oracle Java SEHotspotMultipleYes9.8NetworkLowNoneRequiredChangedHighHighHighOracle Java SE: 8u401, 11.0.22Fixed in 8u401 Patch 123456
      CVE-2024-9001Oracle Java SE, Oracle GraalVM for JDKLibrariesMultipleYes7.5NetworkHighNoneRequiredChangedMediumMediumMediumOracle Java SE: 8u401, 11.0.22; Oracle GraalVM for JDK: 21.3.8, 22.0.0See Note A for mitigation
      +
      +
      + +

      Note A: Apply interim update 22.0.0.1 for GraalVM.

      + + + + diff --git a/src/StellaOps.Feedser.Source.Vndr.Oracle.Tests/Oracle/Fixtures/oracle-detail-cpuapr2024-02.html b/src/StellaOps.Feedser.Source.Vndr.Oracle.Tests/Oracle/Fixtures/oracle-detail-cpuapr2024-02.html new file mode 100644 index 00000000..6b3fef60 --- /dev/null +++ b/src/StellaOps.Feedser.Source.Vndr.Oracle.Tests/Oracle/Fixtures/oracle-detail-cpuapr2024-02.html @@ -0,0 +1,105 @@ + + + Oracle CPU April 2024 Advisory 2 + + + +

      Oracle Security Alert Advisory - April 2024 (CPU02)

      +

      + Mitigations for Oracle WebLogic Server and Oracle Database Server. + Includes references to CVE-2024-9100 with additional product components. +

      + +
      +
      + + + + + + + + + + + + + + + + + +
      Affected Products and VersionsPatch Availability Document
      Oracle WebLogic Server, versions 14.1.1.0.0Fusion Middleware
      Oracle Database Server, versions 19c, 21cDatabase
      +
      +
      + +
      +
      + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
      CVE IDProductComponentProtocolRemote Exploit without Auth.?Base ScoreAttack VectorAttack ComplexPrivs Req'dUser InteractScopeConfidentialityIntegrityAvailabilitySupported Versions AffectedNotes
      CVE-2024-9100Oracle WebLogic ServerConsoleHTTPYes8.1NetworkLowLowRequiredChangedHighHighHighOracle WebLogic Server: 14.1.1.0.0Patch 99999999 available
      CVE-2024-9101Oracle Database ServerSQL*PlusMultipleNo5.4LocalLowLowNoneUnchangedMediumLowLowOracle Database Server: 19c, 21cSee Note B
      +
      +
      + +

      Note B: Customers should review Support Doc 3010101.1 for mitigation guidance.

      + +

      More details at Support KB.

      + + diff --git a/src/StellaOps.Feedser.Source.Vndr.Oracle.Tests/Oracle/Fixtures/oracle-detail-invalid.html b/src/StellaOps.Feedser.Source.Vndr.Oracle.Tests/Oracle/Fixtures/oracle-detail-invalid.html new file mode 100644 index 00000000..c89cc729 --- /dev/null +++ b/src/StellaOps.Feedser.Source.Vndr.Oracle.Tests/Oracle/Fixtures/oracle-detail-invalid.html @@ -0,0 +1,4 @@ + + + + diff --git a/src/StellaOps.Feedser.Source.Vndr.Oracle.Tests/Oracle/OracleConnectorTests.cs b/src/StellaOps.Feedser.Source.Vndr.Oracle.Tests/Oracle/OracleConnectorTests.cs new file mode 100644 index 00000000..3a8e175c --- /dev/null +++ b/src/StellaOps.Feedser.Source.Vndr.Oracle.Tests/Oracle/OracleConnectorTests.cs @@ -0,0 +1,353 @@ +using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using System.Net; +using System.Net.Http; +using System.Net.Http.Headers; +using System.Text; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Http; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Logging.Abstractions; +using Microsoft.Extensions.Options; +using Microsoft.Extensions.Time.Testing; +using MongoDB.Bson; +using MongoDB.Bson.Serialization; +using MongoDB.Bson.Serialization.Serializers; +using MongoDB.Driver; +using StellaOps.Feedser.Models; +using StellaOps.Feedser.Source.Common; +using StellaOps.Feedser.Source.Common.Http; +using StellaOps.Feedser.Source.Common.Testing; +using StellaOps.Feedser.Source.Vndr.Oracle; +using StellaOps.Feedser.Source.Vndr.Oracle.Configuration; +using StellaOps.Feedser.Source.Vndr.Oracle.Internal; +using StellaOps.Feedser.Storage.Mongo; +using StellaOps.Feedser.Storage.Mongo.Advisories; +using StellaOps.Feedser.Storage.Mongo.Documents; +using StellaOps.Feedser.Storage.Mongo.Dtos; +using StellaOps.Feedser.Testing; +using Xunit.Abstractions; + +namespace StellaOps.Feedser.Source.Vndr.Oracle.Tests; + +[Collection("mongo-fixture")] +public sealed class OracleConnectorTests : IAsyncLifetime +{ + private readonly MongoIntegrationFixture _fixture; + private readonly FakeTimeProvider _timeProvider; + private readonly CannedHttpMessageHandler _handler; + private readonly ITestOutputHelper _output; + + private static readonly Uri AdvisoryOne = new("https://www.oracle.com/security-alerts/cpuapr2024-01.html"); + private static readonly Uri AdvisoryTwo = new("https://www.oracle.com/security-alerts/cpuapr2024-02.html"); + private static readonly Uri CalendarUri = new("https://www.oracle.com/security-alerts/cpuapr2024.html"); + + public OracleConnectorTests(MongoIntegrationFixture fixture, ITestOutputHelper output) + { + _fixture = fixture; + _timeProvider = new FakeTimeProvider(new DateTimeOffset(2024, 4, 18, 0, 0, 0, TimeSpan.Zero)); + _handler = new CannedHttpMessageHandler(); + _output = output; + } + + [Fact] + public async Task FetchParseMap_EmitsOraclePsirtSnapshot() + { + await using var provider = await BuildServiceProviderAsync(); + SeedDetails(); + + var calendarFetcher = provider.GetRequiredService(); + var discovered = await calendarFetcher.GetAdvisoryUrisAsync(CancellationToken.None); + _output.WriteLine("Calendar URIs: " + string.Join(", ", discovered.Select(static uri => uri.AbsoluteUri))); + Assert.Equal(2, discovered.Count); + + // Re-seed fixtures because calendar fetch consumes canned responses. + SeedDetails(); + + var connector = provider.GetRequiredService(); + await connector.FetchAsync(provider, CancellationToken.None); + _timeProvider.Advance(TimeSpan.FromMinutes(1)); + await connector.ParseAsync(provider, CancellationToken.None); + await connector.MapAsync(provider, CancellationToken.None); + + var advisoryStore = provider.GetRequiredService(); + var advisories = await advisoryStore.GetRecentAsync(10, CancellationToken.None); + _output.WriteLine("Advisories fetched: " + string.Join(", ", advisories.Select(static a => a.AdvisoryKey))); + _output.WriteLine($"Advisory count: {advisories.Count}"); + Assert.Equal(2, advisories.Count); + + var first = advisories.Single(advisory => advisory.AdvisoryKey == "oracle/cpuapr2024-01-html"); + var second = advisories.Single(advisory => advisory.AdvisoryKey == "oracle/cpuapr2024-02-html"); + Assert.Equal(new DateTimeOffset(2024, 4, 18, 12, 30, 0, TimeSpan.Zero), first.Published); + Assert.Equal(new DateTimeOffset(2024, 4, 19, 8, 15, 0, TimeSpan.Zero), second.Published); + Assert.All(advisories, advisory => + { + Assert.True(advisory.Aliases.Any(alias => alias.StartsWith("CVE-", StringComparison.Ordinal)), $"Expected CVE alias for {advisory.AdvisoryKey}"); + Assert.NotEmpty(advisory.AffectedPackages); + }); + + var snapshot = SnapshotSerializer.ToSnapshot(advisories.OrderBy(static a => a.AdvisoryKey, StringComparer.Ordinal).ToArray()); + var expected = ReadFixture("oracle-advisories.snapshot.json"); + var normalizedSnapshot = Normalize(snapshot); + var normalizedExpected = Normalize(expected); + if (!string.Equals(normalizedExpected, normalizedSnapshot, StringComparison.Ordinal)) + { + var actualPath = Path.Combine(AppContext.BaseDirectory, "Source", "Vndr", "Oracle", "Fixtures", "oracle-advisories.actual.json"); + var actualDirectory = Path.GetDirectoryName(actualPath); + if (!string.IsNullOrEmpty(actualDirectory)) + { + Directory.CreateDirectory(actualDirectory); + } + File.WriteAllText(actualPath, snapshot); + } + + Assert.Equal(normalizedExpected, normalizedSnapshot); + + var psirtCollection = _fixture.Database.GetCollection(MongoStorageDefaults.Collections.PsirtFlags); + var flags = await psirtCollection.Find(Builders.Filter.Empty).ToListAsync(); + _output.WriteLine("Psirt flags: " + string.Join(", ", flags.Select(doc => doc.GetValue("_id", BsonValue.Create("")).ToString()))); + Assert.Equal(2, flags.Count); + Assert.All(flags, doc => Assert.Equal("Oracle", doc["vendor"].AsString)); + } + + [Fact] + public async Task FetchAsync_IdempotentForUnchangedAdvisories() + { + await using var provider = await BuildServiceProviderAsync(); + SeedDetails(); + + var connector = provider.GetRequiredService(); + await connector.FetchAsync(provider, CancellationToken.None); + _timeProvider.Advance(TimeSpan.FromMinutes(1)); + await connector.ParseAsync(provider, CancellationToken.None); + _timeProvider.Advance(TimeSpan.FromMinutes(1)); + await connector.MapAsync(provider, CancellationToken.None); + + // Second run with unchanged documents should rely on fetch cache. + SeedDetails(); + await connector.FetchAsync(provider, CancellationToken.None); + await connector.ParseAsync(provider, CancellationToken.None); + await connector.MapAsync(provider, CancellationToken.None); + + var stateRepository = provider.GetRequiredService(); + var state = await stateRepository.TryGetAsync(VndrOracleConnectorPlugin.SourceName, CancellationToken.None); + Assert.NotNull(state); + var cursor = OracleCursor.FromBson(state!.Cursor); + Assert.Empty(cursor.PendingDocuments); + Assert.Empty(cursor.PendingMappings); + Assert.Equal(2, cursor.FetchCache.Count); + Assert.All(cursor.FetchCache.Values, entry => Assert.False(string.IsNullOrWhiteSpace(entry.Sha256))); + + var documentStore = provider.GetRequiredService(); + var first = await documentStore.FindBySourceAndUriAsync(VndrOracleConnectorPlugin.SourceName, AdvisoryOne.ToString(), CancellationToken.None); + Assert.NotNull(first); + Assert.Equal(DocumentStatuses.Mapped, first!.Status); + + var second = await documentStore.FindBySourceAndUriAsync(VndrOracleConnectorPlugin.SourceName, AdvisoryTwo.ToString(), CancellationToken.None); + Assert.NotNull(second); + Assert.Equal(DocumentStatuses.Mapped, second!.Status); + + var dtoCollection = _fixture.Database.GetCollection(MongoStorageDefaults.Collections.Dto); + var dtoCount = await dtoCollection.CountDocumentsAsync(Builders.Filter.Empty); + Assert.Equal(2, dtoCount); + } + + [Fact] + public async Task FetchAsync_ResumeProcessesNewCalendarEntries() + { + await using var provider = await BuildServiceProviderAsync(); + + AddCalendarResponse(CalendarUri, "oracle-calendar-cpuapr2024-single.html"); + AddDetailResponse(AdvisoryOne, "oracle-detail-cpuapr2024-01.html", "\"oracle-001\""); + + var connector = provider.GetRequiredService(); + await connector.FetchAsync(provider, CancellationToken.None); + _timeProvider.Advance(TimeSpan.FromMinutes(1)); + await connector.ParseAsync(provider, CancellationToken.None); + _timeProvider.Advance(TimeSpan.FromMinutes(1)); + await connector.MapAsync(provider, CancellationToken.None); + + var advisoryStore = provider.GetRequiredService(); + var advisories = await advisoryStore.GetRecentAsync(10, CancellationToken.None); + Assert.Single(advisories); + Assert.Equal("oracle/cpuapr2024-01-html", advisories[0].AdvisoryKey); + + _handler.Clear(); + AddCalendarResponse(CalendarUri, "oracle-calendar-cpuapr2024.html"); + AddDetailResponse(AdvisoryOne, "oracle-detail-cpuapr2024-01.html", "\"oracle-001\""); + AddDetailResponse(AdvisoryTwo, "oracle-detail-cpuapr2024-02.html", "\"oracle-002\""); + + await connector.FetchAsync(provider, CancellationToken.None); + _timeProvider.Advance(TimeSpan.FromMinutes(1)); + await connector.ParseAsync(provider, CancellationToken.None); + _timeProvider.Advance(TimeSpan.FromMinutes(1)); + await connector.MapAsync(provider, CancellationToken.None); + + advisories = await advisoryStore.GetRecentAsync(10, CancellationToken.None); + Assert.Equal(2, advisories.Count); + Assert.Contains(advisories, advisory => advisory.AdvisoryKey == "oracle/cpuapr2024-02-html"); + } + + [Fact] + public async Task ParseAsync_InvalidDocumentIsQuarantined() + { + await using var provider = await BuildServiceProviderAsync(); + + AddCalendarResponse(CalendarUri, "oracle-calendar-cpuapr2024.html"); + AddDetailResponse(AdvisoryOne, "oracle-detail-invalid.html", "\"oracle-001\""); + AddDetailResponse(AdvisoryTwo, "oracle-detail-cpuapr2024-02.html", "\"oracle-002\""); + + var connector = provider.GetRequiredService(); + await connector.FetchAsync(provider, CancellationToken.None); + _timeProvider.Advance(TimeSpan.FromMinutes(1)); + await connector.ParseAsync(provider, CancellationToken.None); + + var documentStore = provider.GetRequiredService(); + var invalidDocument = await documentStore.FindBySourceAndUriAsync(VndrOracleConnectorPlugin.SourceName, AdvisoryOne.ToString(), CancellationToken.None); + Assert.NotNull(invalidDocument); + _output.WriteLine($"Invalid document status: {invalidDocument!.Status}"); + + var rawDoc = await _fixture.Database.GetCollection(MongoStorageDefaults.Collections.Document) + .Find(Builders.Filter.Eq("uri", AdvisoryOne.ToString())) + .FirstOrDefaultAsync(); + if (rawDoc is not null) + { + _output.WriteLine("Raw document: " + rawDoc.ToJson()); + } + + var dtoStore = provider.GetRequiredService(); + var invalidDto = await dtoStore.FindByDocumentIdAsync(invalidDocument.Id, CancellationToken.None); + if (invalidDto is not null) + { + _output.WriteLine("Validation unexpectedly succeeded. DTO: " + invalidDto.Payload.ToJson()); + } + Assert.Equal(DocumentStatuses.Failed, invalidDocument.Status); + Assert.Null(invalidDto); + + var validDocument = await documentStore.FindBySourceAndUriAsync(VndrOracleConnectorPlugin.SourceName, AdvisoryTwo.ToString(), CancellationToken.None); + Assert.NotNull(validDocument); + Assert.Equal(DocumentStatuses.PendingMap, validDocument!.Status); + + _timeProvider.Advance(TimeSpan.FromMinutes(1)); + await connector.MapAsync(provider, CancellationToken.None); + + var advisories = await provider.GetRequiredService().GetRecentAsync(10, CancellationToken.None); + Assert.Single(advisories); + Assert.Equal("oracle/cpuapr2024-02-html", advisories[0].AdvisoryKey); + + var psirtCollection = _fixture.Database.GetCollection(MongoStorageDefaults.Collections.PsirtFlags); + var flagCount = await psirtCollection.CountDocumentsAsync(Builders.Filter.Empty); + Assert.Equal(1, flagCount); + + var stateRepository = provider.GetRequiredService(); + var state = await stateRepository.TryGetAsync(VndrOracleConnectorPlugin.SourceName, CancellationToken.None); + Assert.NotNull(state); + var cursor = OracleCursor.FromBson(state!.Cursor); + Assert.Empty(cursor.PendingDocuments); + Assert.Empty(cursor.PendingMappings); + } + + private async Task BuildServiceProviderAsync() + { + await _fixture.Client.DropDatabaseAsync(_fixture.Database.DatabaseNamespace.DatabaseName); + _handler.Clear(); + + var services = new ServiceCollection(); + services.AddLogging(builder => builder.AddProvider(NullLoggerProvider.Instance)); + services.AddSingleton(_timeProvider); + services.AddSingleton(_handler); + + services.AddMongoStorage(options => + { + options.ConnectionString = _fixture.Runner.ConnectionString; + options.DatabaseName = _fixture.Database.DatabaseNamespace.DatabaseName; + options.CommandTimeout = TimeSpan.FromSeconds(5); + }); + + services.AddSourceCommon(); + services.AddOracleConnector(opts => + { + opts.CalendarUris = new List { CalendarUri }; + opts.RequestDelay = TimeSpan.Zero; + }); + + services.Configure(OracleOptions.HttpClientName, builderOptions => + { + builderOptions.HttpMessageHandlerBuilderActions.Add(builder => + { + builder.PrimaryHandler = _handler; + }); + }); + + var provider = services.BuildServiceProvider(); + var bootstrapper = provider.GetRequiredService(); + await bootstrapper.InitializeAsync(CancellationToken.None); + return provider; + } + + private void SeedDetails() + { + AddCalendarResponse(CalendarUri, "oracle-calendar-cpuapr2024.html"); + AddDetailResponse(AdvisoryOne, "oracle-detail-cpuapr2024-01.html", "\"oracle-001\""); + AddDetailResponse(AdvisoryTwo, "oracle-detail-cpuapr2024-02.html", "\"oracle-002\""); + } + + private void AddCalendarResponse(Uri uri, string fixture) + { + _handler.AddResponse(uri, () => + { + var response = new HttpResponseMessage(HttpStatusCode.OK) + { + Content = new StringContent(ReadFixture(fixture), Encoding.UTF8, "text/html"), + }; + + return response; + }); + } + + private void AddDetailResponse(Uri uri, string fixture, string? etag) + { + _handler.AddResponse(uri, () => + { + var response = new HttpResponseMessage(HttpStatusCode.OK) + { + Content = new StringContent(ReadFixture(fixture), Encoding.UTF8, "text/html"), + }; + + if (!string.IsNullOrEmpty(etag)) + { + response.Headers.ETag = new EntityTagHeaderValue(etag); + } + + return response; + }); + } + + private static string ReadFixture(string filename) + { + var primary = Path.Combine(AppContext.BaseDirectory, "Source", "Vndr", "Oracle", "Fixtures", filename); + if (File.Exists(primary)) + { + return File.ReadAllText(primary); + } + + var fallback = Path.Combine(AppContext.BaseDirectory, "Oracle", "Fixtures", filename); + if (File.Exists(fallback)) + { + return File.ReadAllText(fallback); + } + + throw new FileNotFoundException($"Fixture '{filename}' not found in test output.", filename); + } + + private static string Normalize(string value) + => value.Replace("\r\n", "\n", StringComparison.Ordinal); + + public Task InitializeAsync() => Task.CompletedTask; + + public Task DisposeAsync() => Task.CompletedTask; +} diff --git a/src/StellaOps.Feedser.Source.Vndr.Oracle.Tests/StellaOps.Feedser.Source.Vndr.Oracle.Tests.csproj b/src/StellaOps.Feedser.Source.Vndr.Oracle.Tests/StellaOps.Feedser.Source.Vndr.Oracle.Tests.csproj new file mode 100644 index 00000000..4316bac2 --- /dev/null +++ b/src/StellaOps.Feedser.Source.Vndr.Oracle.Tests/StellaOps.Feedser.Source.Vndr.Oracle.Tests.csproj @@ -0,0 +1,17 @@ + + + net10.0 + enable + enable + + + + + + + + + + + + diff --git a/src/StellaOps.Feedser.Source.Vndr.Oracle/AGENTS.md b/src/StellaOps.Feedser.Source.Vndr.Oracle/AGENTS.md new file mode 100644 index 00000000..0145dc63 --- /dev/null +++ b/src/StellaOps.Feedser.Source.Vndr.Oracle/AGENTS.md @@ -0,0 +1,27 @@ +# AGENTS +## Role +Oracle PSIRT connector for Critical Patch Updates (CPU) and Security Alerts; authoritative vendor ranges and severities for Oracle products; establishes PSIRT precedence over registry or distro where applicable. +## Scope +- Harvest CPU calendar pages and per-advisory content; window by CPU cycle (Jan/Apr/Jul/Oct) and last modified timestamps. +- Validate HTML or JSON; extract CVE lists, affected products, components, versions, fixed patch levels; map to canonical with aliases and psirt_flags. +- Persist raw documents; maintain source_state across cycles; idempotent mapping. +## Participants +- Source.Common (HTTP, validators). +- Storage.Mongo (document, dto, advisory, alias, affected, reference, psirt_flags, source_state). +- Models (canonical; affected ranges for vendor products). +- Core/WebService (jobs: source:oracle:fetch|parse|map). +- Merge engine (later) to prefer PSIRT ranges over NVD for Oracle products. +## Interfaces & contracts +- Alias scheme includes CPU:YYYY-QQ plus individual advisory ids when present; include CVE mappings. +- Affected entries capture product/component and fixedBy patch version; references include product notes and patch docs; kind=advisory or patch. +- Provenance.method=parser; value includes CPU cycle and advisory slug. +## In/Out of scope +In: PSIRT authoritative mapping, cycles handling, precedence signaling. +Out: signing or patch artifact downloads. +## Observability & security expectations +- Metrics: SourceDiagnostics emits `feedser.source.http.*` counters/histograms tagged `feedser.source=oracle`, so observability dashboards slice on that tag to monitor fetch pages, CPU cycle coverage, parse failures, and map affected counts. +- Logs: cycle tags, advisory ids, extraction timings; redact nothing sensitive. +## Tests +- Author and review coverage in `../StellaOps.Feedser.Source.Vndr.Oracle.Tests`. +- Shared fixtures (e.g., `MongoIntegrationFixture`, `ConnectorTestHarness`) live in `../StellaOps.Feedser.Testing`. +- Keep fixtures deterministic; match new cases to real-world advisories or regression scenarios. diff --git a/src/StellaOps.Feedser.Source.Vndr.Oracle/Configuration/OracleOptions.cs b/src/StellaOps.Feedser.Source.Vndr.Oracle/Configuration/OracleOptions.cs new file mode 100644 index 00000000..4a336c7a --- /dev/null +++ b/src/StellaOps.Feedser.Source.Vndr.Oracle/Configuration/OracleOptions.cs @@ -0,0 +1,39 @@ +using System; +using System.Collections.Generic; +using System.Linq; + +namespace StellaOps.Feedser.Source.Vndr.Oracle.Configuration; + +public sealed class OracleOptions +{ + public const string HttpClientName = "vndr-oracle"; + + public List AdvisoryUris { get; set; } = new(); + + public List CalendarUris { get; set; } = new(); + + public TimeSpan RequestDelay { get; set; } = TimeSpan.FromSeconds(1); + + public void Validate() + { + if (AdvisoryUris.Count == 0 && CalendarUris.Count == 0) + { + throw new InvalidOperationException("Oracle connector requires at least one advisory or calendar URI."); + } + + if (AdvisoryUris.Any(uri => uri is null || !uri.IsAbsoluteUri)) + { + throw new InvalidOperationException("All Oracle AdvisoryUris must be absolute URIs."); + } + + if (CalendarUris.Any(uri => uri is null || !uri.IsAbsoluteUri)) + { + throw new InvalidOperationException("All Oracle CalendarUris must be absolute URIs."); + } + + if (RequestDelay < TimeSpan.Zero) + { + throw new InvalidOperationException("RequestDelay cannot be negative."); + } + } +} diff --git a/src/StellaOps.Feedser.Source.Vndr.Oracle/Internal/OracleAffectedEntry.cs b/src/StellaOps.Feedser.Source.Vndr.Oracle/Internal/OracleAffectedEntry.cs new file mode 100644 index 00000000..07804c01 --- /dev/null +++ b/src/StellaOps.Feedser.Source.Vndr.Oracle/Internal/OracleAffectedEntry.cs @@ -0,0 +1,10 @@ +using System.Text.Json.Serialization; + +namespace StellaOps.Feedser.Source.Vndr.Oracle.Internal; + +internal sealed record OracleAffectedEntry( + [property: JsonPropertyName("product")] string Product, + [property: JsonPropertyName("component")] string? Component, + [property: JsonPropertyName("supportedVersions")] string? SupportedVersions, + [property: JsonPropertyName("notes")] string? Notes, + [property: JsonPropertyName("cves")] IReadOnlyList CveIds); diff --git a/src/StellaOps.Feedser.Source.Vndr.Oracle/Internal/OracleCalendarFetcher.cs b/src/StellaOps.Feedser.Source.Vndr.Oracle/Internal/OracleCalendarFetcher.cs new file mode 100644 index 00000000..3dc5406e --- /dev/null +++ b/src/StellaOps.Feedser.Source.Vndr.Oracle/Internal/OracleCalendarFetcher.cs @@ -0,0 +1,92 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Net.Http; +using System.Text.RegularExpressions; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Options; +using StellaOps.Feedser.Source.Vndr.Oracle.Configuration; + +namespace StellaOps.Feedser.Source.Vndr.Oracle.Internal; + +public sealed class OracleCalendarFetcher +{ + private static readonly Regex AnchorRegex = new("]+href=\"(?[^\"]+)\"", RegexOptions.IgnoreCase | RegexOptions.Compiled); + + private readonly IHttpClientFactory _httpClientFactory; + private readonly OracleOptions _options; + private readonly ILogger _logger; + + public OracleCalendarFetcher( + IHttpClientFactory httpClientFactory, + IOptions options, + ILogger logger) + { + _httpClientFactory = httpClientFactory ?? throw new ArgumentNullException(nameof(httpClientFactory)); + _options = (options ?? throw new ArgumentNullException(nameof(options))).Value ?? throw new ArgumentNullException(nameof(options)); + _logger = logger ?? throw new ArgumentNullException(nameof(logger)); + } + + public async Task> GetAdvisoryUrisAsync(CancellationToken cancellationToken) + { + if (_options.CalendarUris.Count == 0) + { + return Array.Empty(); + } + + var discovered = new HashSet(StringComparer.OrdinalIgnoreCase); + var client = _httpClientFactory.CreateClient(OracleOptions.HttpClientName); + + foreach (var calendarUri in _options.CalendarUris) + { + try + { + var content = await client.GetStringAsync(calendarUri, cancellationToken).ConfigureAwait(false); + foreach (var link in ExtractLinks(calendarUri, content)) + { + discovered.Add(link.AbsoluteUri); + } + } + catch (Exception ex) when (ex is HttpRequestException or TaskCanceledException or OperationCanceledException) + { + _logger.LogWarning(ex, "Oracle calendar fetch failed for {Uri}", calendarUri); + } + } + + return discovered + .Select(static uri => new Uri(uri, UriKind.Absolute)) + .OrderBy(static uri => uri.AbsoluteUri, StringComparer.OrdinalIgnoreCase) + .ToArray(); + } + + private static IEnumerable ExtractLinks(Uri baseUri, string html) + { + if (string.IsNullOrWhiteSpace(html)) + { + yield break; + } + + foreach (Match match in AnchorRegex.Matches(html)) + { + if (!match.Success) + { + continue; + } + + var href = match.Groups["url"].Value?.Trim(); + if (string.IsNullOrEmpty(href)) + { + continue; + } + + if (!Uri.TryCreate(baseUri, href, out var uri) || !uri.IsAbsoluteUri) + { + continue; + } + + yield return uri; + } + } +} diff --git a/src/StellaOps.Feedser.Source.Vndr.Oracle/Internal/OracleCursor.cs b/src/StellaOps.Feedser.Source.Vndr.Oracle/Internal/OracleCursor.cs new file mode 100644 index 00000000..72e9b1b4 --- /dev/null +++ b/src/StellaOps.Feedser.Source.Vndr.Oracle/Internal/OracleCursor.cs @@ -0,0 +1,227 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using MongoDB.Bson; +using StellaOps.Feedser.Storage.Mongo.Documents; + +namespace StellaOps.Feedser.Source.Vndr.Oracle.Internal; + +internal sealed record OracleCursor( + DateTimeOffset? LastProcessed, + IReadOnlyCollection PendingDocuments, + IReadOnlyCollection PendingMappings, + IReadOnlyDictionary FetchCache) +{ + private static readonly IReadOnlyCollection EmptyGuidCollection = Array.Empty(); + private static readonly IReadOnlyDictionary EmptyFetchCache = + new Dictionary(StringComparer.OrdinalIgnoreCase); + + public static OracleCursor Empty { get; } = new(null, EmptyGuidCollection, EmptyGuidCollection, EmptyFetchCache); + + public BsonDocument ToBsonDocument() + { + var document = new BsonDocument + { + ["pendingDocuments"] = new BsonArray(PendingDocuments.Select(id => id.ToString())), + ["pendingMappings"] = new BsonArray(PendingMappings.Select(id => id.ToString())), + }; + + if (LastProcessed.HasValue) + { + document["lastProcessed"] = LastProcessed.Value.UtcDateTime; + } + + if (FetchCache.Count > 0) + { + var cacheDocument = new BsonDocument(); + foreach (var (key, entry) in FetchCache) + { + cacheDocument[key] = entry.ToBsonDocument(); + } + + document["fetchCache"] = cacheDocument; + } + + return document; + } + + public static OracleCursor FromBson(BsonDocument? document) + { + if (document is null || document.ElementCount == 0) + { + return Empty; + } + + var lastProcessed = document.TryGetValue("lastProcessed", out var value) + ? ParseDate(value) + : null; + + return new OracleCursor( + lastProcessed, + ReadGuidArray(document, "pendingDocuments"), + ReadGuidArray(document, "pendingMappings"), + ReadFetchCache(document)); + } + + public OracleCursor WithLastProcessed(DateTimeOffset? timestamp) + => this with { LastProcessed = timestamp }; + + public OracleCursor WithPendingDocuments(IEnumerable ids) + => this with { PendingDocuments = ids?.Distinct().ToArray() ?? EmptyGuidCollection }; + + public OracleCursor WithPendingMappings(IEnumerable ids) + => this with { PendingMappings = ids?.Distinct().ToArray() ?? EmptyGuidCollection }; + + public OracleCursor WithFetchCache(IDictionary cache) + { + if (cache is null || cache.Count == 0) + { + return this with { FetchCache = EmptyFetchCache }; + } + + return this with { FetchCache = new Dictionary(cache, StringComparer.OrdinalIgnoreCase) }; + } + + public bool TryGetFetchCache(string key, out OracleFetchCacheEntry entry) + { + if (FetchCache.Count == 0) + { + entry = OracleFetchCacheEntry.Empty; + return false; + } + + return FetchCache.TryGetValue(key, out entry!); + } + + private static DateTimeOffset? ParseDate(BsonValue value) + => value.BsonType switch + { + BsonType.DateTime => DateTime.SpecifyKind(value.ToUniversalTime(), DateTimeKind.Utc), + BsonType.String when DateTimeOffset.TryParse(value.AsString, out var parsed) => parsed.ToUniversalTime(), + _ => null, + }; + + private static IReadOnlyCollection ReadGuidArray(BsonDocument document, string field) + { + if (!document.TryGetValue(field, out var raw) || raw is not BsonArray array) + { + return Array.Empty(); + } + + var result = new List(array.Count); + foreach (var element in array) + { + if (element is null) + { + continue; + } + + if (Guid.TryParse(element.ToString(), out var guid)) + { + result.Add(guid); + } + } + + return result; + } + + private static IReadOnlyDictionary ReadFetchCache(BsonDocument document) + { + if (!document.TryGetValue("fetchCache", out var raw) || raw is not BsonDocument cacheDocument || cacheDocument.ElementCount == 0) + { + return EmptyFetchCache; + } + + var cache = new Dictionary(StringComparer.OrdinalIgnoreCase); + foreach (var element in cacheDocument.Elements) + { + if (element.Value is not BsonDocument entryDocument) + { + continue; + } + + cache[element.Name] = OracleFetchCacheEntry.FromBson(entryDocument); + } + + return cache; + } +} + +internal sealed record OracleFetchCacheEntry(string? Sha256, string? ETag, DateTimeOffset? LastModified) +{ + public static OracleFetchCacheEntry Empty { get; } = new(string.Empty, null, null); + + public BsonDocument ToBsonDocument() + { + var document = new BsonDocument + { + ["sha256"] = Sha256 ?? string.Empty, + }; + + if (!string.IsNullOrWhiteSpace(ETag)) + { + document["etag"] = ETag; + } + + if (LastModified.HasValue) + { + document["lastModified"] = LastModified.Value.UtcDateTime; + } + + return document; + } + + public static OracleFetchCacheEntry FromBson(BsonDocument document) + { + var sha = document.TryGetValue("sha256", out var shaValue) ? shaValue.ToString() : string.Empty; + string? etag = null; + if (document.TryGetValue("etag", out var etagValue) && !etagValue.IsBsonNull) + { + etag = etagValue.ToString(); + } + + DateTimeOffset? lastModified = null; + if (document.TryGetValue("lastModified", out var lastModifiedValue)) + { + lastModified = lastModifiedValue.BsonType switch + { + BsonType.DateTime => DateTime.SpecifyKind(lastModifiedValue.ToUniversalTime(), DateTimeKind.Utc), + BsonType.String when DateTimeOffset.TryParse(lastModifiedValue.AsString, out var parsed) => parsed.ToUniversalTime(), + _ => null, + }; + } + + return new OracleFetchCacheEntry(sha, etag, lastModified); + } + + public static OracleFetchCacheEntry FromDocument(DocumentRecord document) + { + ArgumentNullException.ThrowIfNull(document); + return new OracleFetchCacheEntry( + document.Sha256 ?? string.Empty, + document.Etag, + document.LastModified?.ToUniversalTime()); + } + + public bool Matches(DocumentRecord document) + { + ArgumentNullException.ThrowIfNull(document); + + if (!string.IsNullOrEmpty(Sha256) && !string.IsNullOrEmpty(document.Sha256)) + { + return string.Equals(Sha256, document.Sha256, StringComparison.OrdinalIgnoreCase); + } + + if (!string.IsNullOrEmpty(ETag) && !string.IsNullOrEmpty(document.Etag)) + { + return string.Equals(ETag, document.Etag, StringComparison.Ordinal); + } + + if (LastModified.HasValue && document.LastModified.HasValue) + { + return LastModified.Value.ToUniversalTime() == document.LastModified.Value.ToUniversalTime(); + } + + return false; + } +} diff --git a/src/StellaOps.Feedser.Source.Vndr.Oracle/Internal/OracleDocumentMetadata.cs b/src/StellaOps.Feedser.Source.Vndr.Oracle/Internal/OracleDocumentMetadata.cs new file mode 100644 index 00000000..b609d5fe --- /dev/null +++ b/src/StellaOps.Feedser.Source.Vndr.Oracle/Internal/OracleDocumentMetadata.cs @@ -0,0 +1,56 @@ +using System; +using System.Collections.Generic; +using StellaOps.Feedser.Storage.Mongo.Documents; + +namespace StellaOps.Feedser.Source.Vndr.Oracle.Internal; + +internal sealed record OracleDocumentMetadata( + string AdvisoryId, + string Title, + DateTimeOffset Published, + Uri DetailUri) +{ + private const string AdvisoryIdKey = "oracle.advisoryId"; + private const string TitleKey = "oracle.title"; + private const string PublishedKey = "oracle.published"; + + public static IReadOnlyDictionary CreateMetadata(string advisoryId, string title, DateTimeOffset published) + => new Dictionary(StringComparer.Ordinal) + { + [AdvisoryIdKey] = advisoryId, + [TitleKey] = title, + [PublishedKey] = published.ToString("O"), + }; + + public static OracleDocumentMetadata FromDocument(DocumentRecord document) + { + ArgumentNullException.ThrowIfNull(document); + if (document.Metadata is null) + { + throw new InvalidOperationException("Oracle document metadata missing."); + } + + var metadata = document.Metadata; + if (!metadata.TryGetValue(AdvisoryIdKey, out var advisoryId) || string.IsNullOrWhiteSpace(advisoryId)) + { + throw new InvalidOperationException("Oracle advisory id metadata missing."); + } + + if (!metadata.TryGetValue(TitleKey, out var title) || string.IsNullOrWhiteSpace(title)) + { + throw new InvalidOperationException("Oracle title metadata missing."); + } + + if (!metadata.TryGetValue(PublishedKey, out var publishedRaw) || !DateTimeOffset.TryParse(publishedRaw, out var published)) + { + throw new InvalidOperationException("Oracle published metadata invalid."); + } + + if (!Uri.TryCreate(document.Uri, UriKind.Absolute, out var detailUri)) + { + throw new InvalidOperationException("Oracle document URI invalid."); + } + + return new OracleDocumentMetadata(advisoryId.Trim(), title.Trim(), published.ToUniversalTime(), detailUri); + } +} diff --git a/src/StellaOps.Feedser.Source.Vndr.Oracle/Internal/OracleDto.cs b/src/StellaOps.Feedser.Source.Vndr.Oracle/Internal/OracleDto.cs new file mode 100644 index 00000000..074db653 --- /dev/null +++ b/src/StellaOps.Feedser.Source.Vndr.Oracle/Internal/OracleDto.cs @@ -0,0 +1,16 @@ +using System; +using System.Collections.Generic; +using System.Text.Json.Serialization; + +namespace StellaOps.Feedser.Source.Vndr.Oracle.Internal; + +internal sealed record OracleDto( + [property: JsonPropertyName("advisoryId")] string AdvisoryId, + [property: JsonPropertyName("title")] string Title, + [property: JsonPropertyName("detailUrl")] string DetailUrl, + [property: JsonPropertyName("published")] DateTimeOffset Published, + [property: JsonPropertyName("content")] string Content, + [property: JsonPropertyName("references")] IReadOnlyList References, + [property: JsonPropertyName("cveIds")] IReadOnlyList CveIds, + [property: JsonPropertyName("affected")] IReadOnlyList Affected, + [property: JsonPropertyName("patchDocuments")] IReadOnlyList PatchDocuments); diff --git a/src/StellaOps.Feedser.Source.Vndr.Oracle/Internal/OracleDtoValidator.cs b/src/StellaOps.Feedser.Source.Vndr.Oracle/Internal/OracleDtoValidator.cs new file mode 100644 index 00000000..f782e96b --- /dev/null +++ b/src/StellaOps.Feedser.Source.Vndr.Oracle/Internal/OracleDtoValidator.cs @@ -0,0 +1,276 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text.RegularExpressions; + +namespace StellaOps.Feedser.Source.Vndr.Oracle.Internal; + +internal static class OracleDtoValidator +{ + private const int MaxAdvisoryIdLength = 128; + private const int MaxTitleLength = 512; + private const int MaxContentLength = 200_000; + private const int MaxReferenceCount = 100; + private const int MaxCveCount = 1_024; + private const int MaxAffectedCount = 2_048; + private const int MaxPatchDocumentCount = 512; + private const int MaxProductLength = 512; + private const int MaxComponentLength = 512; + private const int MaxSupportedVersionsLength = 4_096; + private const int MaxNotesLength = 1_024; + private const int MaxPatchTitleLength = 512; + private const int MaxPatchUrlLength = 1_024; + private static readonly Regex CveRegex = new("CVE-\\d{4}-\\d{3,7}", RegexOptions.IgnoreCase | RegexOptions.Compiled); + + public static bool TryNormalize(OracleDto dto, out OracleDto normalized, out string? failureReason) + { + ArgumentNullException.ThrowIfNull(dto); + + failureReason = null; + normalized = dto; + + var advisoryId = dto.AdvisoryId?.Trim(); + if (string.IsNullOrWhiteSpace(advisoryId)) + { + failureReason = "AdvisoryId is required."; + return false; + } + + if (advisoryId.Length > MaxAdvisoryIdLength) + { + failureReason = $"AdvisoryId exceeds {MaxAdvisoryIdLength} characters."; + return false; + } + + var title = string.IsNullOrWhiteSpace(dto.Title) ? advisoryId : dto.Title.Trim(); + if (title.Length > MaxTitleLength) + { + title = title.Substring(0, MaxTitleLength); + } + + var detailUrlRaw = dto.DetailUrl?.Trim(); + if (string.IsNullOrWhiteSpace(detailUrlRaw) || !Uri.TryCreate(detailUrlRaw, UriKind.Absolute, out var detailUri)) + { + failureReason = "DetailUrl must be an absolute URI."; + return false; + } + + if (!string.Equals(detailUri.Scheme, Uri.UriSchemeHttp, StringComparison.OrdinalIgnoreCase) + && !string.Equals(detailUri.Scheme, Uri.UriSchemeHttps, StringComparison.OrdinalIgnoreCase)) + { + failureReason = "DetailUrl must use HTTP or HTTPS."; + return false; + } + + if (dto.Published == default) + { + failureReason = "Published timestamp is required."; + return false; + } + + var published = dto.Published.ToUniversalTime(); + var content = dto.Content?.Trim() ?? string.Empty; + if (string.IsNullOrWhiteSpace(content)) + { + failureReason = "Advisory content is empty."; + return false; + } + + if (content.Length > MaxContentLength) + { + content = content.Substring(0, MaxContentLength); + } + + var references = NormalizeReferences(dto.References); + var cveIds = NormalizeCveIds(dto.CveIds); + var affected = NormalizeAffected(dto.Affected); + var patchDocuments = NormalizePatchDocuments(dto.PatchDocuments); + + normalized = dto with + { + AdvisoryId = advisoryId, + Title = title, + DetailUrl = detailUri.ToString(), + Published = published, + Content = content, + References = references, + CveIds = cveIds, + Affected = affected, + PatchDocuments = patchDocuments, + }; + + return true; + } + + private static IReadOnlyList NormalizeReferences(IReadOnlyList? references) + { + if (references is null || references.Count == 0) + { + return Array.Empty(); + } + + var normalized = new List(Math.Min(references.Count, MaxReferenceCount)); + foreach (var reference in references.Where(static reference => !string.IsNullOrWhiteSpace(reference))) + { + var trimmed = reference.Trim(); + if (Uri.TryCreate(trimmed, UriKind.Absolute, out var uri) + && (string.Equals(uri.Scheme, Uri.UriSchemeHttp, StringComparison.OrdinalIgnoreCase) + || string.Equals(uri.Scheme, Uri.UriSchemeHttps, StringComparison.OrdinalIgnoreCase))) + { + normalized.Add(uri.ToString()); + } + + if (normalized.Count >= MaxReferenceCount) + { + break; + } + } + + if (normalized.Count == 0) + { + return Array.Empty(); + } + + return normalized + .Distinct(StringComparer.OrdinalIgnoreCase) + .OrderBy(static url => url, StringComparer.OrdinalIgnoreCase) + .ToArray(); + } + + private static IReadOnlyList NormalizeCveIds(IReadOnlyList? cveIds) + { + if (cveIds is null || cveIds.Count == 0) + { + return Array.Empty(); + } + + var normalized = new List(Math.Min(cveIds.Count, MaxCveCount)); + foreach (var cve in cveIds.Where(static value => !string.IsNullOrWhiteSpace(value))) + { + var candidate = cve.Trim().ToUpperInvariant(); + if (!CveRegex.IsMatch(candidate)) + { + continue; + } + + normalized.Add(candidate); + if (normalized.Count >= MaxCveCount) + { + break; + } + } + + if (normalized.Count == 0) + { + return Array.Empty(); + } + + return normalized + .Distinct(StringComparer.OrdinalIgnoreCase) + .OrderBy(static value => value, StringComparer.OrdinalIgnoreCase) + .ToArray(); + } + + private static IReadOnlyList NormalizeAffected(IReadOnlyList? entries) + { + if (entries is null || entries.Count == 0) + { + return Array.Empty(); + } + + var normalized = new List(Math.Min(entries.Count, MaxAffectedCount)); + foreach (var entry in entries) + { + if (entry is null) + { + continue; + } + + var product = TrimToLength(entry.Product, MaxProductLength); + if (string.IsNullOrWhiteSpace(product)) + { + continue; + } + + var component = TrimToNull(entry.Component, MaxComponentLength); + var versions = TrimToNull(entry.SupportedVersions, MaxSupportedVersionsLength); + var notes = TrimToNull(entry.Notes, MaxNotesLength); + var cves = NormalizeCveIds(entry.CveIds); + + normalized.Add(new OracleAffectedEntry(product, component, versions, notes, cves)); + if (normalized.Count >= MaxAffectedCount) + { + break; + } + } + + return normalized.Count == 0 ? Array.Empty() : normalized; + } + + private static IReadOnlyList NormalizePatchDocuments(IReadOnlyList? documents) + { + if (documents is null || documents.Count == 0) + { + return Array.Empty(); + } + + var normalized = new List(Math.Min(documents.Count, MaxPatchDocumentCount)); + foreach (var document in documents) + { + if (document is null) + { + continue; + } + + var product = TrimToLength(document.Product, MaxProductLength); + if (string.IsNullOrWhiteSpace(product)) + { + continue; + } + + var title = TrimToNull(document.Title, MaxPatchTitleLength); + var urlRaw = TrimToLength(document.Url, MaxPatchUrlLength); + if (string.IsNullOrWhiteSpace(urlRaw)) + { + continue; + } + + if (!Uri.TryCreate(urlRaw, UriKind.Absolute, out var uri) + || (!string.Equals(uri.Scheme, Uri.UriSchemeHttp, StringComparison.OrdinalIgnoreCase) + && !string.Equals(uri.Scheme, Uri.UriSchemeHttps, StringComparison.OrdinalIgnoreCase))) + { + continue; + } + + normalized.Add(new OraclePatchDocument(product, title, uri.ToString())); + if (normalized.Count >= MaxPatchDocumentCount) + { + break; + } + } + + return normalized.Count == 0 ? Array.Empty() : normalized; + } + + private static string TrimToLength(string? value, int maxLength) + { + if (string.IsNullOrWhiteSpace(value)) + { + return string.Empty; + } + + var trimmed = value.Trim(); + if (trimmed.Length <= maxLength) + { + return trimmed; + } + + return trimmed[..maxLength]; + } + + private static string? TrimToNull(string? value, int maxLength) + { + var trimmed = TrimToLength(value, maxLength); + return string.IsNullOrWhiteSpace(trimmed) ? null : trimmed; + } +} diff --git a/src/StellaOps.Feedser.Source.Vndr.Oracle/Internal/OracleMapper.cs b/src/StellaOps.Feedser.Source.Vndr.Oracle/Internal/OracleMapper.cs new file mode 100644 index 00000000..0a805658 --- /dev/null +++ b/src/StellaOps.Feedser.Source.Vndr.Oracle/Internal/OracleMapper.cs @@ -0,0 +1,426 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text.RegularExpressions; +using StellaOps.Feedser.Models; +using StellaOps.Feedser.Source.Common.Packages; +using StellaOps.Feedser.Storage.Mongo.Documents; +using StellaOps.Feedser.Storage.Mongo.Dtos; +using StellaOps.Feedser.Storage.Mongo.PsirtFlags; + +namespace StellaOps.Feedser.Source.Vndr.Oracle.Internal; + +internal static class OracleMapper +{ + private static readonly Regex FixedVersionRegex = new("(?:Fixed|Fix)\\s+(?:in|available in|for)\\s+(?[A-Za-z0-9._-]+)", RegexOptions.IgnoreCase | RegexOptions.Compiled); + private static readonly Regex PatchNumberRegex = new("Patch\\s+(?\\d+)", RegexOptions.IgnoreCase | RegexOptions.Compiled); + + public static (Advisory Advisory, PsirtFlagRecord Flag) Map( + OracleDto dto, + DocumentRecord document, + DtoRecord dtoRecord, + string sourceName, + DateTimeOffset mappedAt) + { + ArgumentNullException.ThrowIfNull(dto); + ArgumentNullException.ThrowIfNull(document); + ArgumentNullException.ThrowIfNull(dtoRecord); + ArgumentException.ThrowIfNullOrEmpty(sourceName); + + var advisoryKey = $"oracle/{dto.AdvisoryId}"; + var fetchProvenance = new AdvisoryProvenance(sourceName, "document", document.Uri, document.FetchedAt.ToUniversalTime()); + var mappingProvenance = new AdvisoryProvenance(sourceName, "mapping", dto.AdvisoryId, mappedAt.ToUniversalTime()); + + var aliases = BuildAliases(dto); + var references = BuildReferences(dto, sourceName, mappedAt); + var affectedPackages = BuildAffectedPackages(dto, sourceName, mappedAt); + + var advisory = new Advisory( + advisoryKey, + dto.Title, + dto.Content, + language: "en", + published: dto.Published.ToUniversalTime(), + modified: null, + severity: null, + exploitKnown: false, + aliases, + references, + affectedPackages, + Array.Empty(), + new[] { fetchProvenance, mappingProvenance }); + + var flag = new PsirtFlagRecord( + advisoryKey, + "Oracle", + sourceName, + dto.AdvisoryId, + mappedAt.ToUniversalTime()); + + return (advisory, flag); + } + + private static IReadOnlyList BuildAliases(OracleDto dto) + { + var aliases = new HashSet(StringComparer.OrdinalIgnoreCase) + { + $"ORACLE:{dto.AdvisoryId}".ToUpperInvariant(), + }; + + foreach (var cve in dto.CveIds) + { + if (!string.IsNullOrWhiteSpace(cve)) + { + aliases.Add(cve.Trim().ToUpperInvariant()); + } + } + + return aliases + .OrderBy(static alias => alias, StringComparer.OrdinalIgnoreCase) + .ToArray(); + } + + private static IReadOnlyList BuildReferences(OracleDto dto, string sourceName, DateTimeOffset recordedAt) + { + var comparer = StringComparer.OrdinalIgnoreCase; + var entries = new List<(AdvisoryReference Reference, int Priority)> + { + (new AdvisoryReference( + dto.DetailUrl, + "advisory", + "oracle", + dto.Title, + new AdvisoryProvenance(sourceName, "reference", dto.DetailUrl, recordedAt.ToUniversalTime())), 0), + }; + + foreach (var document in dto.PatchDocuments) + { + var summary = document.Title ?? document.Product; + entries.Add((new AdvisoryReference( + document.Url, + "patch", + "oracle", + summary, + new AdvisoryProvenance(sourceName, "reference", document.Url, recordedAt.ToUniversalTime())), 1)); + } + + foreach (var url in dto.References) + { + entries.Add((new AdvisoryReference( + url, + "reference", + null, + null, + new AdvisoryProvenance(sourceName, "reference", url, recordedAt.ToUniversalTime())), 2)); + } + + foreach (var cve in dto.CveIds) + { + if (string.IsNullOrWhiteSpace(cve)) + { + continue; + } + + var cveUrl = $"https://www.cve.org/CVERecord?id={cve}"; + entries.Add((new AdvisoryReference( + cveUrl, + "advisory", + cve, + null, + new AdvisoryProvenance(sourceName, "reference", cveUrl, recordedAt.ToUniversalTime())), 3)); + } + + return entries + .GroupBy(tuple => tuple.Reference.Url, comparer) + .Select(group => group + .OrderBy(t => t.Priority) + .ThenBy(t => t.Reference.Kind ?? string.Empty, comparer) + .ThenBy(t => t.Reference.Url, comparer) + .First()) + .OrderBy(t => t.Priority) + .ThenBy(t => t.Reference.Kind ?? string.Empty, comparer) + .ThenBy(t => t.Reference.Url, comparer) + .Select(t => t.Reference) + .ToArray(); + } + + private static IReadOnlyList BuildAffectedPackages(OracleDto dto, string sourceName, DateTimeOffset recordedAt) + { + if (dto.Affected.Count == 0) + { + return Array.Empty(); + } + + var packages = new List(); + var seen = new HashSet(StringComparer.OrdinalIgnoreCase); + + foreach (var entry in dto.Affected) + { + if (entry is null) + { + continue; + } + + var component = NormalizeComponent(entry.Component); + var notes = entry.Notes; + + foreach (var segment in SplitSupportedVersions(entry.Product, entry.SupportedVersions)) + { + if (string.IsNullOrWhiteSpace(segment.Product)) + { + continue; + } + + var identifier = CreateIdentifier(segment.Product, component); + var baseExpression = segment.Versions ?? entry.SupportedVersions ?? string.Empty; + var composedExpression = baseExpression; + + if (!string.IsNullOrEmpty(notes)) + { + composedExpression = string.IsNullOrEmpty(composedExpression) + ? $"notes: {notes}" + : $"{composedExpression} (notes: {notes})"; + } + + var rangeExpression = string.IsNullOrWhiteSpace(composedExpression) ? null : composedExpression; + var (fixedVersion, patchNumber) = ExtractFixMetadata(notes); + var rangeProvenance = new AdvisoryProvenance(sourceName, "range", identifier, recordedAt.ToUniversalTime()); + var rangePrimitives = BuildVendorRangePrimitives(entry, segment, component, baseExpression, rangeExpression, notes, fixedVersion, patchNumber); + + var ranges = rangeExpression is null && string.IsNullOrEmpty(fixedVersion) + ? Array.Empty() + : new[] + { + new AffectedVersionRange( + rangeKind: "vendor", + introducedVersion: null, + fixedVersion: fixedVersion, + lastAffectedVersion: null, + rangeExpression: rangeExpression, + provenance: rangeProvenance, + primitives: rangePrimitives), + }; + + var provenance = new[] + { + new AdvisoryProvenance(sourceName, "affected", identifier, recordedAt.ToUniversalTime()), + }; + + var package = new AffectedPackage( + AffectedPackageTypes.Vendor, + identifier, + component, + ranges, + statuses: Array.Empty(), + provenance: provenance); + + var key = $"{identifier}::{component}::{ranges.FirstOrDefault()?.CreateDeterministicKey()}"; + if (seen.Add(key)) + { + packages.Add(package); + } + } + } + + return packages.Count == 0 ? Array.Empty() : packages; + } + + private static IEnumerable<(string Product, string? Versions)> SplitSupportedVersions(string product, string? supportedVersions) + { + var normalizedProduct = string.IsNullOrWhiteSpace(product) ? "Oracle Product" : product.Trim(); + + if (string.IsNullOrWhiteSpace(supportedVersions)) + { + yield return (normalizedProduct, null); + yield break; + } + + var segments = supportedVersions.Split(';', StringSplitOptions.RemoveEmptyEntries | StringSplitOptions.TrimEntries); + if (segments.Length <= 1) + { + yield return (normalizedProduct, supportedVersions.Trim()); + yield break; + } + + foreach (var segment in segments) + { + var text = segment.Trim(); + if (text.Length == 0) + { + continue; + } + + var colonIndex = text.IndexOf(':'); + if (colonIndex > 0) + { + var name = text[..colonIndex].Trim(); + var versions = text[(colonIndex + 1)..].Trim(); + yield return (string.IsNullOrEmpty(name) ? normalizedProduct : name, versions); + } + else + { + yield return (normalizedProduct, text); + } + } + } + + private static RangePrimitives? BuildVendorRangePrimitives( + OracleAffectedEntry entry, + (string Product, string? Versions) segment, + string? component, + string? baseExpression, + string? rangeExpression, + string? notes, + string? fixedVersion, + string? patchNumber) + { + var extensions = new Dictionary(StringComparer.Ordinal); + + AddExtension(extensions, "oracle.product", segment.Product); + AddExtension(extensions, "oracle.productRaw", entry.Product); + AddExtension(extensions, "oracle.component", component); + AddExtension(extensions, "oracle.componentRaw", entry.Component); + AddExtension(extensions, "oracle.segmentVersions", segment.Versions); + AddExtension(extensions, "oracle.supportedVersions", entry.SupportedVersions); + AddExtension(extensions, "oracle.rangeExpression", rangeExpression); + AddExtension(extensions, "oracle.baseExpression", baseExpression); + AddExtension(extensions, "oracle.notes", notes); + AddExtension(extensions, "oracle.fixedVersion", fixedVersion); + AddExtension(extensions, "oracle.patchNumber", patchNumber); + + var versionTokens = ExtractVersionTokens(baseExpression); + if (versionTokens.Count > 0) + { + extensions["oracle.versionTokens"] = string.Join('|', versionTokens); + + var normalizedTokens = versionTokens + .Select(NormalizeSemVerToken) + .Where(static token => !string.IsNullOrEmpty(token)) + .Cast() + .Distinct(StringComparer.Ordinal) + .ToArray(); + + if (normalizedTokens.Length > 0) + { + extensions["oracle.versionTokens.normalized"] = string.Join('|', normalizedTokens); + } + } + + if (extensions.Count == 0) + { + return null; + } + + return new RangePrimitives(null, null, null, extensions); + } + + private static IReadOnlyList ExtractVersionTokens(string? baseExpression) + { + if (string.IsNullOrWhiteSpace(baseExpression)) + { + return Array.Empty(); + } + + var tokens = new List(); + foreach (var token in baseExpression.Split(new[] { ',', ';' }, StringSplitOptions.RemoveEmptyEntries)) + { + var value = token.Trim(); + if (value.Length == 0 || !value.Any(char.IsDigit)) + { + continue; + } + + tokens.Add(value); + } + + return tokens.Count == 0 ? Array.Empty() : tokens; + } + + private static string? NormalizeSemVerToken(string token) + { + if (string.IsNullOrWhiteSpace(token)) + { + return null; + } + + if (PackageCoordinateHelper.TryParseSemVer(token, out _, out var normalized) && !string.IsNullOrWhiteSpace(normalized)) + { + return normalized; + } + + if (Version.TryParse(token, out var parsed)) + { + if (parsed.Build >= 0 && parsed.Revision >= 0) + { + return $"{parsed.Major}.{parsed.Minor}.{parsed.Build}.{parsed.Revision}"; + } + + if (parsed.Build >= 0) + { + return $"{parsed.Major}.{parsed.Minor}.{parsed.Build}"; + } + + return $"{parsed.Major}.{parsed.Minor}"; + } + + return null; + } + + private static void AddExtension(Dictionary extensions, string key, string? value) + { + if (string.IsNullOrWhiteSpace(value)) + { + return; + } + + extensions[key] = value.Trim(); + } + + private static string? NormalizeComponent(string? component) + { + if (string.IsNullOrWhiteSpace(component)) + { + return null; + } + + var trimmed = component.Trim(); + return trimmed.Length == 0 ? null : trimmed; + } + + private static string CreateIdentifier(string product, string? component) + { + var normalizedProduct = product.Trim(); + if (string.IsNullOrEmpty(component)) + { + return normalizedProduct; + } + + return $"{normalizedProduct}::{component}"; + } + + private static (string? FixedVersion, string? PatchNumber) ExtractFixMetadata(string? notes) + { + if (string.IsNullOrWhiteSpace(notes)) + { + return (null, null); + } + + string? fixedVersion = null; + string? patchNumber = null; + + var match = FixedVersionRegex.Match(notes); + if (match.Success) + { + fixedVersion = match.Groups["value"].Value.Trim(); + } + + match = PatchNumberRegex.Match(notes); + if (match.Success) + { + patchNumber = match.Groups["value"].Value.Trim(); + fixedVersion ??= patchNumber; + } + + return (fixedVersion, patchNumber); + } +} diff --git a/src/StellaOps.Feedser.Source.Vndr.Oracle/Internal/OracleParser.cs b/src/StellaOps.Feedser.Source.Vndr.Oracle/Internal/OracleParser.cs new file mode 100644 index 00000000..6cd27135 --- /dev/null +++ b/src/StellaOps.Feedser.Source.Vndr.Oracle/Internal/OracleParser.cs @@ -0,0 +1,457 @@ +using System; +using System.Collections.Generic; +using System.Globalization; +using System.Linq; +using System.Text.RegularExpressions; +using AngleSharp.Html.Dom; +using AngleSharp.Html.Parser; + +namespace StellaOps.Feedser.Source.Vndr.Oracle.Internal; + +internal static class OracleParser +{ + private static readonly Regex AnchorRegex = new("]+href=\"(?https?://[^\"]+)\"", RegexOptions.IgnoreCase | RegexOptions.Compiled); + private static readonly Regex TagRegex = new("<[^>]+>", RegexOptions.Compiled); + private static readonly Regex WhitespaceRegex = new("\\s+", RegexOptions.Compiled); + private static readonly Regex CveRegex = new("CVE-\\d{4}-\\d{3,7}", RegexOptions.IgnoreCase | RegexOptions.Compiled); + private static readonly Regex UpdatedDateRegex = new("\"updatedDate\"\\s*:\\s*\"(?[^\"]+)\"", RegexOptions.IgnoreCase | RegexOptions.Compiled); + private static readonly string[] AllowedReferenceTokens = + { + "security-alerts", + "/kb/", + "/patches", + "/rs", + "/support/", + "/mos/", + "/technicalresources/", + "/technetwork/" + }; + + public static OracleDto Parse(string html, OracleDocumentMetadata metadata) + { + ArgumentException.ThrowIfNullOrEmpty(html); + ArgumentNullException.ThrowIfNull(metadata); + + var parser = new HtmlParser(); + var document = parser.ParseDocument(html); + + var published = ExtractPublishedDate(document) ?? metadata.Published; + var content = Sanitize(html); + var affected = ExtractAffectedEntries(document); + var references = ExtractReferences(html); + var patchDocuments = ExtractPatchDocuments(document, metadata.DetailUri); + var cveIds = ExtractCveIds(document, content, affected); + + return new OracleDto( + metadata.AdvisoryId, + metadata.Title, + metadata.DetailUri.ToString(), + published, + content, + references, + cveIds, + affected, + patchDocuments); + } + + private static string Sanitize(string html) + { + var withoutTags = TagRegex.Replace(html, " "); + var decoded = System.Net.WebUtility.HtmlDecode(withoutTags) ?? string.Empty; + return WhitespaceRegex.Replace(decoded, " ").Trim(); + } + + private static IReadOnlyList ExtractReferences(string html) + { + var references = new HashSet(StringComparer.OrdinalIgnoreCase); + foreach (Match match in AnchorRegex.Matches(html)) + { + if (!match.Success) + { + continue; + } + + var raw = match.Groups["url"].Value?.Trim(); + if (string.IsNullOrEmpty(raw)) + { + continue; + } + + var decoded = System.Net.WebUtility.HtmlDecode(raw) ?? raw; + + if (!Uri.TryCreate(decoded, UriKind.Absolute, out var uri)) + { + continue; + } + + if (!string.Equals(uri.Scheme, Uri.UriSchemeHttp, StringComparison.OrdinalIgnoreCase) + && !string.Equals(uri.Scheme, Uri.UriSchemeHttps, StringComparison.OrdinalIgnoreCase)) + { + continue; + } + + if (!ShouldIncludeReference(uri)) + { + continue; + } + + references.Add(uri.ToString()); + } + + return references.Count == 0 + ? Array.Empty() + : references.OrderBy(url => url, StringComparer.OrdinalIgnoreCase).ToArray(); + } + + private static bool ShouldIncludeReference(Uri uri) + { + if (uri.Host.EndsWith("cve.org", StringComparison.OrdinalIgnoreCase)) + { + return true; + } + + if (!uri.Host.EndsWith("oracle.com", StringComparison.OrdinalIgnoreCase)) + { + return false; + } + + if (uri.Query.Contains("type=doc", StringComparison.OrdinalIgnoreCase)) + { + return true; + } + + var path = uri.AbsolutePath ?? string.Empty; + return AllowedReferenceTokens.Any(token => path.Contains(token, StringComparison.OrdinalIgnoreCase)); + } + + private static DateTimeOffset? ExtractPublishedDate(IHtmlDocument document) + { + var meta = document.QuerySelectorAll("meta") + .FirstOrDefault(static element => string.Equals(element.GetAttribute("name"), "Updated Date", StringComparison.OrdinalIgnoreCase)); + if (meta is not null && TryParseOracleDate(meta.GetAttribute("content"), out var parsed)) + { + return parsed; + } + + foreach (var script in document.Scripts) + { + var text = script.TextContent; + if (string.IsNullOrWhiteSpace(text)) + { + continue; + } + + var match = UpdatedDateRegex.Match(text); + if (!match.Success) + { + continue; + } + + if (TryParseOracleDate(match.Groups["value"].Value, out var embedded)) + { + return embedded; + } + } + + return null; + } + + private static bool TryParseOracleDate(string? value, out DateTimeOffset result) + { + if (!string.IsNullOrWhiteSpace(value) + && DateTimeOffset.TryParse(value, CultureInfo.InvariantCulture, DateTimeStyles.AssumeUniversal, out result)) + { + result = result.ToUniversalTime(); + return true; + } + + result = default; + return false; + } + + private static IReadOnlyList ExtractAffectedEntries(IHtmlDocument document) + { + var entries = new List(); + + foreach (var table in document.QuerySelectorAll("table")) + { + if (table is not IHtmlTableElement tableElement) + { + continue; + } + + if (!IsRiskMatrixTable(tableElement)) + { + continue; + } + + var lastProduct = string.Empty; + var lastComponent = string.Empty; + var lastVersions = string.Empty; + var lastNotes = string.Empty; + IReadOnlyList lastCves = Array.Empty(); + + foreach (var body in tableElement.Bodies) + { + foreach (var row in body.Rows) + { + if (row is not IHtmlTableRowElement tableRow || tableRow.Cells.Length == 0) + { + continue; + } + + var cveText = NormalizeCellText(GetCellText(tableRow, 0)); + var cves = ExtractCvesFromText(cveText); + if (cves.Count == 0 && lastCves.Count > 0) + { + cves = lastCves; + } + else if (cves.Count > 0) + { + lastCves = cves; + } + + var product = NormalizeCellText(GetCellText(tableRow, 1)); + if (string.IsNullOrEmpty(product)) + { + product = lastProduct; + } + else + { + lastProduct = product; + } + + var component = NormalizeCellText(GetCellText(tableRow, 2)); + if (string.IsNullOrEmpty(component)) + { + component = lastComponent; + } + else + { + lastComponent = component; + } + + var supportedVersions = NormalizeCellText(GetCellTextFromEnd(tableRow, 2)); + if (string.IsNullOrEmpty(supportedVersions)) + { + supportedVersions = lastVersions; + } + else + { + lastVersions = supportedVersions; + } + + var notes = NormalizeCellText(GetCellTextFromEnd(tableRow, 1)); + if (string.IsNullOrEmpty(notes)) + { + notes = lastNotes; + } + else + { + lastNotes = notes; + } + + if (string.IsNullOrEmpty(product) || cves.Count == 0) + { + continue; + } + + entries.Add(new OracleAffectedEntry( + product, + string.IsNullOrEmpty(component) ? null : component, + string.IsNullOrEmpty(supportedVersions) ? null : supportedVersions, + string.IsNullOrEmpty(notes) ? null : notes, + cves)); + } + } + } + + return entries.Count == 0 ? Array.Empty() : entries; + } + + private static IReadOnlyList ExtractCveIds(IHtmlDocument document, string content, IReadOnlyList affectedEntries) + { + var cves = new HashSet(StringComparer.OrdinalIgnoreCase); + + if (!string.IsNullOrWhiteSpace(content)) + { + foreach (Match match in CveRegex.Matches(content)) + { + cves.Add(match.Value.ToUpperInvariant()); + } + } + + foreach (var entry in affectedEntries) + { + foreach (var cve in entry.CveIds) + { + if (!string.IsNullOrWhiteSpace(cve)) + { + cves.Add(cve.ToUpperInvariant()); + } + } + } + + var bodyText = document.Body?.TextContent; + if (!string.IsNullOrWhiteSpace(bodyText)) + { + foreach (Match match in CveRegex.Matches(bodyText)) + { + cves.Add(match.Value.ToUpperInvariant()); + } + } + + return cves.Count == 0 + ? Array.Empty() + : cves.OrderBy(static id => id, StringComparer.OrdinalIgnoreCase).ToArray(); + } + + private static IReadOnlyList ExtractPatchDocuments(IHtmlDocument document, Uri detailUri) + { + var results = new List(); + + foreach (var table in document.QuerySelectorAll("table")) + { + if (table is not IHtmlTableElement tableElement) + { + continue; + } + + if (!TableHasPatchHeader(tableElement)) + { + continue; + } + + foreach (var body in tableElement.Bodies) + { + foreach (var row in body.Rows) + { + if (row is not IHtmlTableRowElement tableRow || tableRow.Cells.Length < 2) + { + continue; + } + + var product = NormalizeCellText(tableRow.Cells[0]?.TextContent); + if (string.IsNullOrEmpty(product)) + { + continue; + } + + var anchor = tableRow.Cells[1]?.QuerySelector("a"); + if (anchor is null) + { + continue; + } + + var href = anchor.GetAttribute("href"); + if (string.IsNullOrWhiteSpace(href)) + { + continue; + } + + var decoded = System.Net.WebUtility.HtmlDecode(href) ?? href; + + if (!Uri.TryCreate(detailUri, decoded, out var uri) || !uri.IsAbsoluteUri) + { + continue; + } + + if (!string.Equals(uri.Scheme, Uri.UriSchemeHttp, StringComparison.OrdinalIgnoreCase) + && !string.Equals(uri.Scheme, Uri.UriSchemeHttps, StringComparison.OrdinalIgnoreCase)) + { + continue; + } + + var title = NormalizeCellText(anchor.TextContent); + results.Add(new OraclePatchDocument(product, string.IsNullOrEmpty(title) ? null : title, uri.ToString())); + } + } + } + + return results.Count == 0 ? Array.Empty() : results; + } + + private static bool IsRiskMatrixTable(IHtmlTableElement table) + { + var headerText = table.Head?.TextContent; + if (string.IsNullOrWhiteSpace(headerText)) + { + return false; + } + + return headerText.Contains("CVE ID", StringComparison.OrdinalIgnoreCase) + && headerText.Contains("Supported Versions", StringComparison.OrdinalIgnoreCase); + } + + private static bool TableHasPatchHeader(IHtmlTableElement table) + { + var headerText = table.Head?.TextContent; + if (string.IsNullOrWhiteSpace(headerText)) + { + return false; + } + + return headerText.Contains("Affected Products and Versions", StringComparison.OrdinalIgnoreCase) + && headerText.Contains("Patch Availability Document", StringComparison.OrdinalIgnoreCase); + } + + private static string? GetCellText(IHtmlTableRowElement row, int index) + { + if (index < 0 || index >= row.Cells.Length) + { + return null; + } + + return row.Cells[index]?.TextContent; + } + + private static string? GetCellTextFromEnd(IHtmlTableRowElement row, int offsetFromEnd) + { + if (offsetFromEnd <= 0) + { + return null; + } + + var index = row.Cells.Length - offsetFromEnd; + return index >= 0 ? row.Cells[index]?.TextContent : null; + } + + private static IReadOnlyList ExtractCvesFromText(string? text) + { + if (string.IsNullOrWhiteSpace(text)) + { + return Array.Empty(); + } + + var matches = CveRegex.Matches(text); + if (matches.Count == 0) + { + return Array.Empty(); + } + + var set = new HashSet(StringComparer.OrdinalIgnoreCase); + foreach (Match match in matches) + { + if (match.Success) + { + set.Add(match.Value.ToUpperInvariant()); + } + } + + return set.Count == 0 + ? Array.Empty() + : set.OrderBy(static id => id, StringComparer.Ordinal).ToArray(); + } + + private static string NormalizeCellText(string? value) + { + if (string.IsNullOrWhiteSpace(value)) + { + return string.Empty; + } + + var cleaned = value.Replace('\u00A0', ' '); + cleaned = WhitespaceRegex.Replace(cleaned, " "); + return cleaned.Trim(); + } +} diff --git a/src/StellaOps.Feedser.Source.Vndr.Oracle/Internal/OraclePatchDocument.cs b/src/StellaOps.Feedser.Source.Vndr.Oracle/Internal/OraclePatchDocument.cs new file mode 100644 index 00000000..42787aeb --- /dev/null +++ b/src/StellaOps.Feedser.Source.Vndr.Oracle/Internal/OraclePatchDocument.cs @@ -0,0 +1,8 @@ +using System.Text.Json.Serialization; + +namespace StellaOps.Feedser.Source.Vndr.Oracle.Internal; + +internal sealed record OraclePatchDocument( + [property: JsonPropertyName("product")] string Product, + [property: JsonPropertyName("title")] string? Title, + [property: JsonPropertyName("url")] string Url); diff --git a/src/StellaOps.Feedser.Source.Vndr.Oracle/Jobs.cs b/src/StellaOps.Feedser.Source.Vndr.Oracle/Jobs.cs new file mode 100644 index 00000000..ba7602be --- /dev/null +++ b/src/StellaOps.Feedser.Source.Vndr.Oracle/Jobs.cs @@ -0,0 +1,46 @@ +using System; +using System.Threading; +using System.Threading.Tasks; +using StellaOps.Feedser.Core.Jobs; + +namespace StellaOps.Feedser.Source.Vndr.Oracle; + +internal static class OracleJobKinds +{ + public const string Fetch = "source:vndr-oracle:fetch"; + public const string Parse = "source:vndr-oracle:parse"; + public const string Map = "source:vndr-oracle:map"; +} + +internal sealed class OracleFetchJob : IJob +{ + private readonly OracleConnector _connector; + + public OracleFetchJob(OracleConnector connector) + => _connector = connector ?? throw new ArgumentNullException(nameof(connector)); + + public Task ExecuteAsync(JobExecutionContext context, CancellationToken cancellationToken) + => _connector.FetchAsync(context.Services, cancellationToken); +} + +internal sealed class OracleParseJob : IJob +{ + private readonly OracleConnector _connector; + + public OracleParseJob(OracleConnector connector) + => _connector = connector ?? throw new ArgumentNullException(nameof(connector)); + + public Task ExecuteAsync(JobExecutionContext context, CancellationToken cancellationToken) + => _connector.ParseAsync(context.Services, cancellationToken); +} + +internal sealed class OracleMapJob : IJob +{ + private readonly OracleConnector _connector; + + public OracleMapJob(OracleConnector connector) + => _connector = connector ?? throw new ArgumentNullException(nameof(connector)); + + public Task ExecuteAsync(JobExecutionContext context, CancellationToken cancellationToken) + => _connector.MapAsync(context.Services, cancellationToken); +} diff --git a/src/StellaOps.Feedser.Source.Vndr.Oracle/OracleConnector.cs b/src/StellaOps.Feedser.Source.Vndr.Oracle/OracleConnector.cs new file mode 100644 index 00000000..54e24a8f --- /dev/null +++ b/src/StellaOps.Feedser.Source.Vndr.Oracle/OracleConnector.cs @@ -0,0 +1,366 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text.Json; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Options; +using MongoDB.Bson; +using StellaOps.Feedser.Source.Common; +using StellaOps.Feedser.Source.Common.Fetch; +using StellaOps.Feedser.Source.Vndr.Oracle.Configuration; +using StellaOps.Feedser.Source.Vndr.Oracle.Internal; +using StellaOps.Feedser.Storage.Mongo; +using StellaOps.Feedser.Storage.Mongo.Advisories; +using StellaOps.Feedser.Storage.Mongo.Documents; +using StellaOps.Feedser.Storage.Mongo.Dtos; +using StellaOps.Feedser.Storage.Mongo.PsirtFlags; +using StellaOps.Plugin; + +namespace StellaOps.Feedser.Source.Vndr.Oracle; + +public sealed class OracleConnector : IFeedConnector +{ + private static readonly JsonSerializerOptions SerializerOptions = new() + { + PropertyNamingPolicy = JsonNamingPolicy.CamelCase, + DefaultIgnoreCondition = System.Text.Json.Serialization.JsonIgnoreCondition.WhenWritingNull, + }; + + private readonly SourceFetchService _fetchService; + private readonly RawDocumentStorage _rawDocumentStorage; + private readonly IDocumentStore _documentStore; + private readonly IDtoStore _dtoStore; + private readonly IAdvisoryStore _advisoryStore; + private readonly IPsirtFlagStore _psirtFlagStore; + private readonly ISourceStateRepository _stateRepository; + private readonly OracleCalendarFetcher _calendarFetcher; + private readonly OracleOptions _options; + private readonly TimeProvider _timeProvider; + private readonly ILogger _logger; + + public OracleConnector( + SourceFetchService fetchService, + RawDocumentStorage rawDocumentStorage, + IDocumentStore documentStore, + IDtoStore dtoStore, + IAdvisoryStore advisoryStore, + IPsirtFlagStore psirtFlagStore, + ISourceStateRepository stateRepository, + OracleCalendarFetcher calendarFetcher, + IOptions options, + TimeProvider? timeProvider, + ILogger logger) + { + _fetchService = fetchService ?? throw new ArgumentNullException(nameof(fetchService)); + _rawDocumentStorage = rawDocumentStorage ?? throw new ArgumentNullException(nameof(rawDocumentStorage)); + _documentStore = documentStore ?? throw new ArgumentNullException(nameof(documentStore)); + _dtoStore = dtoStore ?? throw new ArgumentNullException(nameof(dtoStore)); + _advisoryStore = advisoryStore ?? throw new ArgumentNullException(nameof(advisoryStore)); + _psirtFlagStore = psirtFlagStore ?? throw new ArgumentNullException(nameof(psirtFlagStore)); + _stateRepository = stateRepository ?? throw new ArgumentNullException(nameof(stateRepository)); + _calendarFetcher = calendarFetcher ?? throw new ArgumentNullException(nameof(calendarFetcher)); + _options = (options ?? throw new ArgumentNullException(nameof(options))).Value ?? throw new ArgumentNullException(nameof(options)); + _options.Validate(); + _timeProvider = timeProvider ?? TimeProvider.System; + _logger = logger ?? throw new ArgumentNullException(nameof(logger)); + } + + public string SourceName => VndrOracleConnectorPlugin.SourceName; + + public async Task FetchAsync(IServiceProvider services, CancellationToken cancellationToken) + { + var cursor = await GetCursorAsync(cancellationToken).ConfigureAwait(false); + var pendingDocuments = cursor.PendingDocuments.ToList(); + var pendingMappings = cursor.PendingMappings.ToList(); + var fetchCache = new Dictionary(cursor.FetchCache, StringComparer.OrdinalIgnoreCase); + var touchedResources = new HashSet(StringComparer.OrdinalIgnoreCase); + var now = _timeProvider.GetUtcNow(); + + var advisoryUris = await ResolveAdvisoryUrisAsync(cancellationToken).ConfigureAwait(false); + + foreach (var uri in advisoryUris) + { + cancellationToken.ThrowIfCancellationRequested(); + + try + { + var cacheKey = uri.AbsoluteUri; + touchedResources.Add(cacheKey); + + var advisoryId = DeriveAdvisoryId(uri); + var title = advisoryId.Replace('-', ' '); + var published = now; + + var metadata = OracleDocumentMetadata.CreateMetadata(advisoryId, title, published); + var existing = await _documentStore.FindBySourceAndUriAsync(SourceName, uri.ToString(), cancellationToken).ConfigureAwait(false); + + var request = new SourceFetchRequest(OracleOptions.HttpClientName, SourceName, uri) + { + Metadata = metadata, + ETag = existing?.Etag, + LastModified = existing?.LastModified, + AcceptHeaders = new[] { "text/html", "application/xhtml+xml", "text/plain;q=0.5" }, + }; + + var result = await _fetchService.FetchAsync(request, cancellationToken).ConfigureAwait(false); + if (!result.IsSuccess || result.Document is null) + { + continue; + } + + var cacheEntry = OracleFetchCacheEntry.FromDocument(result.Document); + if (existing is not null + && string.Equals(existing.Status, DocumentStatuses.Mapped, StringComparison.Ordinal) + && cursor.TryGetFetchCache(cacheKey, out var cached) + && cached.Matches(result.Document)) + { + _logger.LogDebug("Oracle advisory {AdvisoryId} unchanged; skipping parse/map", advisoryId); + await _documentStore.UpdateStatusAsync(result.Document.Id, existing.Status, cancellationToken).ConfigureAwait(false); + pendingDocuments.Remove(result.Document.Id); + pendingMappings.Remove(result.Document.Id); + fetchCache[cacheKey] = cacheEntry; + continue; + } + + fetchCache[cacheKey] = cacheEntry; + + if (!pendingDocuments.Contains(result.Document.Id)) + { + pendingDocuments.Add(result.Document.Id); + } + + if (_options.RequestDelay > TimeSpan.Zero) + { + await Task.Delay(_options.RequestDelay, cancellationToken).ConfigureAwait(false); + } + } + catch (Exception ex) + { + _logger.LogError(ex, "Oracle fetch failed for {Uri}", uri); + await _stateRepository.MarkFailureAsync(SourceName, _timeProvider.GetUtcNow(), TimeSpan.FromMinutes(10), ex.Message, cancellationToken).ConfigureAwait(false); + throw; + } + } + + if (fetchCache.Count > 0 && touchedResources.Count > 0) + { + var stale = fetchCache.Keys.Where(key => !touchedResources.Contains(key)).ToArray(); + foreach (var key in stale) + { + fetchCache.Remove(key); + } + } + + var updatedCursor = cursor + .WithPendingDocuments(pendingDocuments) + .WithPendingMappings(pendingMappings) + .WithFetchCache(fetchCache) + .WithLastProcessed(now); + + await UpdateCursorAsync(updatedCursor, cancellationToken).ConfigureAwait(false); + } + + public async Task ParseAsync(IServiceProvider services, CancellationToken cancellationToken) + { + var cursor = await GetCursorAsync(cancellationToken).ConfigureAwait(false); + if (cursor.PendingDocuments.Count == 0) + { + return; + } + + var pendingDocuments = cursor.PendingDocuments.ToList(); + var pendingMappings = cursor.PendingMappings.ToList(); + + foreach (var documentId in cursor.PendingDocuments) + { + cancellationToken.ThrowIfCancellationRequested(); + + var document = await _documentStore.FindAsync(documentId, cancellationToken).ConfigureAwait(false); + if (document is null) + { + pendingDocuments.Remove(documentId); + pendingMappings.Remove(documentId); + continue; + } + + if (!document.GridFsId.HasValue) + { + _logger.LogWarning("Oracle document {DocumentId} missing GridFS payload", document.Id); + await _documentStore.UpdateStatusAsync(document.Id, DocumentStatuses.Failed, cancellationToken).ConfigureAwait(false); + pendingDocuments.Remove(documentId); + pendingMappings.Remove(documentId); + continue; + } + + OracleDto dto; + try + { + var metadata = OracleDocumentMetadata.FromDocument(document); + var content = await _rawDocumentStorage.DownloadAsync(document.GridFsId.Value, cancellationToken).ConfigureAwait(false); + var html = System.Text.Encoding.UTF8.GetString(content); + dto = OracleParser.Parse(html, metadata); + } + catch (Exception ex) + { + _logger.LogError(ex, "Oracle parse failed for document {DocumentId}", document.Id); + await _documentStore.UpdateStatusAsync(document.Id, DocumentStatuses.Failed, cancellationToken).ConfigureAwait(false); + pendingDocuments.Remove(documentId); + pendingMappings.Remove(documentId); + continue; + } + + if (!OracleDtoValidator.TryNormalize(dto, out var normalized, out var validationError)) + { + _logger.LogWarning("Oracle validation failed for document {DocumentId}: {Reason}", document.Id, validationError ?? "unknown"); + await _documentStore.UpdateStatusAsync(document.Id, DocumentStatuses.Failed, cancellationToken).ConfigureAwait(false); + pendingDocuments.Remove(documentId); + pendingMappings.Remove(documentId); + continue; + } + + dto = normalized; + + var json = JsonSerializer.Serialize(dto, SerializerOptions); + var payload = BsonDocument.Parse(json); + var validatedAt = _timeProvider.GetUtcNow(); + + var existingDto = await _dtoStore.FindByDocumentIdAsync(document.Id, cancellationToken).ConfigureAwait(false); + var dtoRecord = existingDto is null + ? new DtoRecord(Guid.NewGuid(), document.Id, SourceName, "oracle.advisory.v1", payload, validatedAt) + : existingDto with + { + Payload = payload, + SchemaVersion = "oracle.advisory.v1", + ValidatedAt = validatedAt, + }; + + await _dtoStore.UpsertAsync(dtoRecord, cancellationToken).ConfigureAwait(false); + await _documentStore.UpdateStatusAsync(document.Id, DocumentStatuses.PendingMap, cancellationToken).ConfigureAwait(false); + + pendingDocuments.Remove(documentId); + if (!pendingMappings.Contains(documentId)) + { + pendingMappings.Add(documentId); + } + } + + var updatedCursor = cursor + .WithPendingDocuments(pendingDocuments) + .WithPendingMappings(pendingMappings); + + await UpdateCursorAsync(updatedCursor, cancellationToken).ConfigureAwait(false); + } + + public async Task MapAsync(IServiceProvider services, CancellationToken cancellationToken) + { + var cursor = await GetCursorAsync(cancellationToken).ConfigureAwait(false); + if (cursor.PendingMappings.Count == 0) + { + return; + } + + var pendingMappings = cursor.PendingMappings.ToList(); + + foreach (var documentId in cursor.PendingMappings) + { + cancellationToken.ThrowIfCancellationRequested(); + + var dtoRecord = await _dtoStore.FindByDocumentIdAsync(documentId, cancellationToken).ConfigureAwait(false); + var document = await _documentStore.FindAsync(documentId, cancellationToken).ConfigureAwait(false); + + if (dtoRecord is null || document is null) + { + pendingMappings.Remove(documentId); + continue; + } + + OracleDto? dto; + try + { + var json = dtoRecord.Payload.ToJson(); + dto = JsonSerializer.Deserialize(json, SerializerOptions); + } + catch (Exception ex) + { + _logger.LogError(ex, "Oracle DTO deserialization failed for document {DocumentId}", documentId); + await _documentStore.UpdateStatusAsync(document.Id, DocumentStatuses.Failed, cancellationToken).ConfigureAwait(false); + pendingMappings.Remove(documentId); + continue; + } + + if (dto is null) + { + _logger.LogWarning("Oracle DTO payload deserialized as null for document {DocumentId}", documentId); + await _documentStore.UpdateStatusAsync(document.Id, DocumentStatuses.Failed, cancellationToken).ConfigureAwait(false); + pendingMappings.Remove(documentId); + continue; + } + + var mappedAt = _timeProvider.GetUtcNow(); + var (advisory, flag) = OracleMapper.Map(dto, document, dtoRecord, SourceName, mappedAt); + await _advisoryStore.UpsertAsync(advisory, cancellationToken).ConfigureAwait(false); + await _psirtFlagStore.UpsertAsync(flag, cancellationToken).ConfigureAwait(false); + await _documentStore.UpdateStatusAsync(document.Id, DocumentStatuses.Mapped, cancellationToken).ConfigureAwait(false); + + pendingMappings.Remove(documentId); + } + + var updatedCursor = cursor.WithPendingMappings(pendingMappings); + await UpdateCursorAsync(updatedCursor, cancellationToken).ConfigureAwait(false); + } + + private async Task GetCursorAsync(CancellationToken cancellationToken) + { + var record = await _stateRepository.TryGetAsync(SourceName, cancellationToken).ConfigureAwait(false); + return OracleCursor.FromBson(record?.Cursor); + } + + private async Task UpdateCursorAsync(OracleCursor cursor, CancellationToken cancellationToken) + { + var completedAt = _timeProvider.GetUtcNow(); + await _stateRepository.UpdateCursorAsync(SourceName, cursor.ToBsonDocument(), completedAt, cancellationToken).ConfigureAwait(false); + } + + private async Task> ResolveAdvisoryUrisAsync(CancellationToken cancellationToken) + { + var uris = new HashSet(StringComparer.OrdinalIgnoreCase); + + foreach (var uri in _options.AdvisoryUris) + { + if (uri is not null) + { + uris.Add(uri.AbsoluteUri); + } + } + + var calendarUris = await _calendarFetcher.GetAdvisoryUrisAsync(cancellationToken).ConfigureAwait(false); + foreach (var uri in calendarUris) + { + uris.Add(uri.AbsoluteUri); + } + + return uris + .Select(static value => new Uri(value, UriKind.Absolute)) + .OrderBy(static value => value.AbsoluteUri, StringComparer.OrdinalIgnoreCase) + .ToArray(); + } + + private static string DeriveAdvisoryId(Uri uri) + { + var segments = uri.Segments; + if (segments.Length == 0) + { + return uri.AbsoluteUri; + } + + var slug = segments[^1].Trim('/'); + if (string.IsNullOrWhiteSpace(slug)) + { + return uri.AbsoluteUri; + } + + return slug.Replace('.', '-'); + } +} diff --git a/src/StellaOps.Feedser.Source.Vndr.Oracle/OracleDependencyInjectionRoutine.cs b/src/StellaOps.Feedser.Source.Vndr.Oracle/OracleDependencyInjectionRoutine.cs new file mode 100644 index 00000000..1a1f42a2 --- /dev/null +++ b/src/StellaOps.Feedser.Source.Vndr.Oracle/OracleDependencyInjectionRoutine.cs @@ -0,0 +1,54 @@ +using System; +using Microsoft.Extensions.Configuration; +using Microsoft.Extensions.DependencyInjection; +using StellaOps.DependencyInjection; +using StellaOps.Feedser.Core.Jobs; +using StellaOps.Feedser.Source.Vndr.Oracle.Configuration; + +namespace StellaOps.Feedser.Source.Vndr.Oracle; + +public sealed class OracleDependencyInjectionRoutine : IDependencyInjectionRoutine +{ + private const string ConfigurationSection = "feedser:sources:oracle"; + + public IServiceCollection Register(IServiceCollection services, IConfiguration configuration) + { + ArgumentNullException.ThrowIfNull(services); + ArgumentNullException.ThrowIfNull(configuration); + + services.AddOracleConnector(options => + { + configuration.GetSection(ConfigurationSection).Bind(options); + options.Validate(); + }); + + services.AddTransient(); + services.AddTransient(); + services.AddTransient(); + + services.PostConfigure(options => + { + EnsureJob(options, OracleJobKinds.Fetch, typeof(OracleFetchJob)); + EnsureJob(options, OracleJobKinds.Parse, typeof(OracleParseJob)); + EnsureJob(options, OracleJobKinds.Map, typeof(OracleMapJob)); + }); + + return services; + } + + private static void EnsureJob(JobSchedulerOptions options, string kind, Type jobType) + { + if (options.Definitions.ContainsKey(kind)) + { + return; + } + + options.Definitions[kind] = new JobDefinition( + kind, + jobType, + options.DefaultTimeout, + options.DefaultLeaseDuration, + CronExpression: null, + Enabled: true); + } +} diff --git a/src/StellaOps.Feedser.Source.Vndr.Oracle/OracleServiceCollectionExtensions.cs b/src/StellaOps.Feedser.Source.Vndr.Oracle/OracleServiceCollectionExtensions.cs new file mode 100644 index 00000000..1acaaca8 --- /dev/null +++ b/src/StellaOps.Feedser.Source.Vndr.Oracle/OracleServiceCollectionExtensions.cs @@ -0,0 +1,42 @@ +using System; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.DependencyInjection.Extensions; +using Microsoft.Extensions.Options; +using StellaOps.Feedser.Source.Common.Http; +using StellaOps.Feedser.Source.Vndr.Oracle.Configuration; +using StellaOps.Feedser.Source.Vndr.Oracle.Internal; + +namespace StellaOps.Feedser.Source.Vndr.Oracle; + +public static class OracleServiceCollectionExtensions +{ + public static IServiceCollection AddOracleConnector(this IServiceCollection services, Action configure) + { + ArgumentNullException.ThrowIfNull(services); + ArgumentNullException.ThrowIfNull(configure); + + services.AddOptions() + .Configure(configure) + .PostConfigure(static opts => opts.Validate()); + + services.AddSourceHttpClient(OracleOptions.HttpClientName, static (sp, clientOptions) => + { + var options = sp.GetRequiredService>().Value; + clientOptions.Timeout = TimeSpan.FromSeconds(30); + clientOptions.UserAgent = "StellaOps.Feedser.Oracle/1.0"; + clientOptions.AllowedHosts.Clear(); + foreach (var uri in options.AdvisoryUris) + { + clientOptions.AllowedHosts.Add(uri.Host); + } + foreach (var uri in options.CalendarUris) + { + clientOptions.AllowedHosts.Add(uri.Host); + } + }); + + services.AddTransient(); + services.AddTransient(); + return services; + } +} diff --git a/src/StellaOps.Feedser.Source.Vndr.Oracle/Properties/AssemblyInfo.cs b/src/StellaOps.Feedser.Source.Vndr.Oracle/Properties/AssemblyInfo.cs new file mode 100644 index 00000000..b773c235 --- /dev/null +++ b/src/StellaOps.Feedser.Source.Vndr.Oracle/Properties/AssemblyInfo.cs @@ -0,0 +1,3 @@ +using System.Runtime.CompilerServices; + +[assembly: InternalsVisibleTo("StellaOps.Feedser.Source.Vndr.Oracle.Tests")] diff --git a/src/StellaOps.Feedser.Source.Vndr.Oracle/StellaOps.Feedser.Source.Vndr.Oracle.csproj b/src/StellaOps.Feedser.Source.Vndr.Oracle/StellaOps.Feedser.Source.Vndr.Oracle.csproj new file mode 100644 index 00000000..75922ca4 --- /dev/null +++ b/src/StellaOps.Feedser.Source.Vndr.Oracle/StellaOps.Feedser.Source.Vndr.Oracle.csproj @@ -0,0 +1,17 @@ + + + + net10.0 + enable + enable + + + + + + + + + + + diff --git a/src/StellaOps.Feedser.Source.Vndr.Oracle/TASKS.md b/src/StellaOps.Feedser.Source.Vndr.Oracle/TASKS.md new file mode 100644 index 00000000..7b670e97 --- /dev/null +++ b/src/StellaOps.Feedser.Source.Vndr.Oracle/TASKS.md @@ -0,0 +1,13 @@ +# TASKS +| Task | Owner(s) | Depends on | Notes | +|---|---|---|---| +|Oracle options & HttpClient configuration|BE-Conn-Oracle|Source.Common|**DONE** – `AddOracleConnector` wires options and allowlisted HttpClient.| +|CPU calendar plus advisory fetchers|BE-Conn-Oracle|Source.Common|**DONE** – resume/backfill scenario covered with new integration test and fetch cache pruning verified.| +|Extractor for products/components/fix levels|BE-Conn-Oracle|Source.Common|**DONE** – HTML risk matrices parsed into vendor packages with fix heuristics and normalized versions.| +|DTO schema and validation|BE-Conn-Oracle, QA|Source.Common|**DONE** – `OracleDtoValidator` enforces required fields and quarantines malformed payloads.| +|Canonical mapping with psirt_flags|BE-Conn-Oracle|Models|**DONE** – mapper now emits CVE aliases, patch references, and vendor affected packages under psirt flag provenance.| +|SourceState and dedupe|BE-Conn-Oracle|Storage.Mongo|**DONE** – cursor fetch cache tracks SHA/ETag to skip unchanged advisories and clear pending work.| +|Golden fixtures and precedence tests (later with merge)|QA|Source.Vndr.Oracle|**DONE** – snapshot fixtures and psirt flag assertions added in `OracleConnectorTests`.| +|Dependency injection routine & job registration|BE-Conn-Oracle|Core|**DONE** – `OracleDependencyInjectionRoutine` registers connector and fetch/parse/map jobs with scheduler defaults.| +|Implement Oracle connector skeleton|BE-Conn-Oracle|Source.Common|**DONE** – fetch/parse/map pipeline persists documents, DTOs, advisories, psirt flags.| +|Range primitives & provenance backfill|BE-Conn-Oracle|Models, Storage.Mongo|**DONE** – vendor primitives emitted (extensions + fix parsing), provenance tagging/logging extended, snapshots refreshed.| diff --git a/src/StellaOps.Feedser.Source.Vndr.Oracle/VndrOracleConnectorPlugin.cs b/src/StellaOps.Feedser.Source.Vndr.Oracle/VndrOracleConnectorPlugin.cs new file mode 100644 index 00000000..0ec2ee31 --- /dev/null +++ b/src/StellaOps.Feedser.Source.Vndr.Oracle/VndrOracleConnectorPlugin.cs @@ -0,0 +1,21 @@ +using System; +using Microsoft.Extensions.DependencyInjection; +using StellaOps.Plugin; + +namespace StellaOps.Feedser.Source.Vndr.Oracle; + +public sealed class VndrOracleConnectorPlugin : IConnectorPlugin +{ + public const string SourceName = "vndr-oracle"; + + public string Name => SourceName; + + public bool IsAvailable(IServiceProvider services) + => services.GetService() is not null; + + public IFeedConnector Create(IServiceProvider services) + { + ArgumentNullException.ThrowIfNull(services); + return services.GetRequiredService(); + } +} diff --git a/src/StellaOps.Feedser.Source.Vndr.Vmware.Tests/StellaOps.Feedser.Source.Vndr.Vmware.Tests.csproj b/src/StellaOps.Feedser.Source.Vndr.Vmware.Tests/StellaOps.Feedser.Source.Vndr.Vmware.Tests.csproj new file mode 100644 index 00000000..88018948 --- /dev/null +++ b/src/StellaOps.Feedser.Source.Vndr.Vmware.Tests/StellaOps.Feedser.Source.Vndr.Vmware.Tests.csproj @@ -0,0 +1,18 @@ + + + net10.0 + enable + enable + + + + + + + + + + PreserveNewest + + + diff --git a/src/StellaOps.Feedser.Source.Vndr.Vmware.Tests/Vmware/Fixtures/vmware-advisories.snapshot.json b/src/StellaOps.Feedser.Source.Vndr.Vmware.Tests/Vmware/Fixtures/vmware-advisories.snapshot.json new file mode 100644 index 00000000..361d8f63 --- /dev/null +++ b/src/StellaOps.Feedser.Source.Vndr.Vmware.Tests/Vmware/Fixtures/vmware-advisories.snapshot.json @@ -0,0 +1,275 @@ +[ + { + "advisoryKey": "VMSA-2024-0001", + "affectedPackages": [ + { + "identifier": "VMware ESXi 7.0", + "platform": null, + "provenance": [ + { + "fieldMask": [], + "kind": "affected", + "recordedAt": "2024-04-05T00:00:00+00:00", + "source": "vmware", + "value": "VMware ESXi 7.0" + } + ], + "statuses": [], + "type": "vendor", + "versionRanges": [ + { + "fixedVersion": "7.0u3f", + "introducedVersion": "7.0", + "lastAffectedVersion": null, + "primitives": { + "evr": null, + "hasVendorExtensions": true, + "nevra": null, + "semVer": { + "constraintExpression": null, + "fixed": null, + "fixedInclusive": false, + "introduced": "7.0", + "introducedInclusive": true, + "lastAffected": null, + "lastAffectedInclusive": false + }, + "vendorExtensions": { + "vmware.product": "VMware ESXi 7.0", + "vmware.version.raw": "7.0", + "vmware.fixedVersion.raw": "7.0u3f" + } + }, + "provenance": { + "fieldMask": [], + "kind": "range", + "recordedAt": "2024-04-05T00:00:00+00:00", + "source": "vmware", + "value": "VMware ESXi 7.0" + }, + "rangeExpression": "7.0", + "rangeKind": "vendor" + } + ] + }, + { + "identifier": "VMware vCenter Server 8.0", + "platform": null, + "provenance": [ + { + "fieldMask": [], + "kind": "affected", + "recordedAt": "2024-04-05T00:00:00+00:00", + "source": "vmware", + "value": "VMware vCenter Server 8.0" + } + ], + "statuses": [], + "type": "vendor", + "versionRanges": [ + { + "fixedVersion": "8.0a", + "introducedVersion": "8.0", + "lastAffectedVersion": null, + "primitives": { + "evr": null, + "hasVendorExtensions": true, + "nevra": null, + "semVer": { + "constraintExpression": null, + "fixed": null, + "fixedInclusive": false, + "introduced": "8.0", + "introducedInclusive": true, + "lastAffected": null, + "lastAffectedInclusive": false + }, + "vendorExtensions": { + "vmware.product": "VMware vCenter Server 8.0", + "vmware.version.raw": "8.0", + "vmware.fixedVersion.raw": "8.0a" + } + }, + "provenance": { + "fieldMask": [], + "kind": "range", + "recordedAt": "2024-04-05T00:00:00+00:00", + "source": "vmware", + "value": "VMware vCenter Server 8.0" + }, + "rangeExpression": "8.0", + "rangeKind": "vendor" + } + ] + } + ], + "aliases": [ + "CVE-2024-1000", + "CVE-2024-1001", + "VMSA-2024-0001" + ], + "cvssMetrics": [], + "exploitKnown": false, + "language": "en", + "modified": "2024-04-01T10:00:00+00:00", + "provenance": [ + { + "fieldMask": [], + "kind": "document", + "recordedAt": "2024-04-05T00:00:00+00:00", + "source": "vmware", + "value": "https://vmware.example/api/vmsa/VMSA-2024-0001.json" + }, + { + "fieldMask": [], + "kind": "mapping", + "recordedAt": "2024-04-05T00:00:00+00:00", + "source": "vmware", + "value": "VMSA-2024-0001" + } + ], + "published": "2024-04-01T10:00:00+00:00", + "references": [ + { + "kind": "kb", + "provenance": { + "fieldMask": [], + "kind": "reference", + "recordedAt": "2024-04-05T00:00:00+00:00", + "source": "vmware", + "value": "https://kb.vmware.example/90234" + }, + "sourceTag": "kb", + "summary": null, + "url": "https://kb.vmware.example/90234" + }, + { + "kind": "advisory", + "provenance": { + "fieldMask": [], + "kind": "reference", + "recordedAt": "2024-04-05T00:00:00+00:00", + "source": "vmware", + "value": "https://www.vmware.com/security/advisories/VMSA-2024-0001.html" + }, + "sourceTag": "advisory", + "summary": null, + "url": "https://www.vmware.com/security/advisories/VMSA-2024-0001.html" + } + ], + "severity": null, + "summary": "Security updates for VMware ESXi 7.0 and vCenter Server 8.0 resolve multiple vulnerabilities.", + "title": "VMware ESXi and vCenter Server updates address vulnerabilities" + }, + { + "advisoryKey": "VMSA-2024-0002", + "affectedPackages": [ + { + "identifier": "VMware Cloud Foundation 5.x", + "platform": null, + "provenance": [ + { + "fieldMask": [], + "kind": "affected", + "recordedAt": "2024-04-05T00:00:00+00:00", + "source": "vmware", + "value": "VMware Cloud Foundation 5.x" + } + ], + "statuses": [], + "type": "vendor", + "versionRanges": [ + { + "fixedVersion": "5.1.1", + "introducedVersion": "5.1", + "lastAffectedVersion": null, + "primitives": { + "evr": null, + "hasVendorExtensions": true, + "nevra": null, + "semVer": { + "constraintExpression": null, + "fixed": "5.1.1", + "fixedInclusive": false, + "introduced": "5.1", + "introducedInclusive": true, + "lastAffected": null, + "lastAffectedInclusive": false + }, + "vendorExtensions": { + "vmware.product": "VMware Cloud Foundation 5.x", + "vmware.version.raw": "5.1", + "vmware.fixedVersion.raw": "5.1.1" + } + }, + "provenance": { + "fieldMask": [], + "kind": "range", + "recordedAt": "2024-04-05T00:00:00+00:00", + "source": "vmware", + "value": "VMware Cloud Foundation 5.x" + }, + "rangeExpression": "5.1", + "rangeKind": "vendor" + } + ] + } + ], + "aliases": [ + "CVE-2024-2000", + "VMSA-2024-0002" + ], + "cvssMetrics": [], + "exploitKnown": false, + "language": "en", + "modified": "2024-04-02T09:00:00+00:00", + "provenance": [ + { + "fieldMask": [], + "kind": "document", + "recordedAt": "2024-04-05T00:00:00+00:00", + "source": "vmware", + "value": "https://vmware.example/api/vmsa/VMSA-2024-0002.json" + }, + { + "fieldMask": [], + "kind": "mapping", + "recordedAt": "2024-04-05T00:00:00+00:00", + "source": "vmware", + "value": "VMSA-2024-0002" + } + ], + "published": "2024-04-02T09:00:00+00:00", + "references": [ + { + "kind": "kb", + "provenance": { + "fieldMask": [], + "kind": "reference", + "recordedAt": "2024-04-05T00:00:00+00:00", + "source": "vmware", + "value": "https://kb.vmware.example/91234" + }, + "sourceTag": "kb", + "summary": null, + "url": "https://kb.vmware.example/91234" + }, + { + "kind": "advisory", + "provenance": { + "fieldMask": [], + "kind": "reference", + "recordedAt": "2024-04-05T00:00:00+00:00", + "source": "vmware", + "value": "https://www.vmware.com/security/advisories/VMSA-2024-0002.html" + }, + "sourceTag": "advisory", + "summary": null, + "url": "https://www.vmware.com/security/advisories/VMSA-2024-0002.html" + } + ], + "severity": null, + "summary": "An update is available for VMware Cloud Foundation components to address a remote code execution vulnerability.", + "title": "VMware Cloud Foundation remote code execution vulnerability" + } +] \ No newline at end of file diff --git a/src/StellaOps.Feedser.Source.Vndr.Vmware.Tests/Vmware/Fixtures/vmware-detail-vmsa-2024-0001.json b/src/StellaOps.Feedser.Source.Vndr.Vmware.Tests/Vmware/Fixtures/vmware-detail-vmsa-2024-0001.json new file mode 100644 index 00000000..3a5ceb52 --- /dev/null +++ b/src/StellaOps.Feedser.Source.Vndr.Vmware.Tests/Vmware/Fixtures/vmware-detail-vmsa-2024-0001.json @@ -0,0 +1,33 @@ +{ + "id": "VMSA-2024-0001", + "title": "VMware ESXi and vCenter Server updates address vulnerabilities", + "summary": "Security updates for VMware ESXi 7.0 and vCenter Server 8.0 resolve multiple vulnerabilities.", + "published": "2024-04-01T10:00:00Z", + "modified": "2024-04-01T10:00:00Z", + "cves": [ + "CVE-2024-1000", + "CVE-2024-1001" + ], + "affected": [ + { + "product": "VMware ESXi 7.0", + "version": "7.0", + "fixedVersion": "7.0u3f" + }, + { + "product": "VMware vCenter Server 8.0", + "version": "8.0", + "fixedVersion": "8.0a" + } + ], + "references": [ + { + "type": "kb", + "url": "https://kb.vmware.example/90234" + }, + { + "type": "advisory", + "url": "https://www.vmware.com/security/advisories/VMSA-2024-0001.html" + } + ] +} diff --git a/src/StellaOps.Feedser.Source.Vndr.Vmware.Tests/Vmware/Fixtures/vmware-detail-vmsa-2024-0002.json b/src/StellaOps.Feedser.Source.Vndr.Vmware.Tests/Vmware/Fixtures/vmware-detail-vmsa-2024-0002.json new file mode 100644 index 00000000..1210de9f --- /dev/null +++ b/src/StellaOps.Feedser.Source.Vndr.Vmware.Tests/Vmware/Fixtures/vmware-detail-vmsa-2024-0002.json @@ -0,0 +1,27 @@ +{ + "id": "VMSA-2024-0002", + "title": "VMware Cloud Foundation remote code execution vulnerability", + "summary": "An update is available for VMware Cloud Foundation components to address a remote code execution vulnerability.", + "published": "2024-04-02T09:00:00Z", + "modified": "2024-04-02T09:00:00Z", + "cves": [ + "CVE-2024-2000" + ], + "affected": [ + { + "product": "VMware Cloud Foundation 5.x", + "version": "5.1", + "fixedVersion": "5.1.1" + } + ], + "references": [ + { + "type": "kb", + "url": "https://kb.vmware.example/91234" + }, + { + "type": "advisory", + "url": "https://www.vmware.com/security/advisories/VMSA-2024-0002.html" + } + ] +} diff --git a/src/StellaOps.Feedser.Source.Vndr.Vmware.Tests/Vmware/Fixtures/vmware-detail-vmsa-2024-0003.json b/src/StellaOps.Feedser.Source.Vndr.Vmware.Tests/Vmware/Fixtures/vmware-detail-vmsa-2024-0003.json new file mode 100644 index 00000000..8980c163 --- /dev/null +++ b/src/StellaOps.Feedser.Source.Vndr.Vmware.Tests/Vmware/Fixtures/vmware-detail-vmsa-2024-0003.json @@ -0,0 +1,23 @@ +{ + "id": "VMSA-2024-0003", + "title": "VMware NSX-T advisory for input validation issue", + "summary": "VMware NSX-T has resolved an input validation vulnerability impacting API endpoints.", + "published": "2024-04-03T08:15:00Z", + "modified": "2024-04-03T08:15:00Z", + "cves": [ + "CVE-2024-3000" + ], + "affected": [ + { + "product": "VMware NSX-T 3.2", + "version": "3.2", + "fixedVersion": "3.2.3" + } + ], + "references": [ + { + "type": "kb", + "url": "https://kb.vmware.example/93456" + } + ] +} diff --git a/src/StellaOps.Feedser.Source.Vndr.Vmware.Tests/Vmware/Fixtures/vmware-index-initial.json b/src/StellaOps.Feedser.Source.Vndr.Vmware.Tests/Vmware/Fixtures/vmware-index-initial.json new file mode 100644 index 00000000..3ef9a8f9 --- /dev/null +++ b/src/StellaOps.Feedser.Source.Vndr.Vmware.Tests/Vmware/Fixtures/vmware-index-initial.json @@ -0,0 +1,12 @@ +[ + { + "id": "VMSA-2024-0001", + "url": "https://vmware.example/api/vmsa/VMSA-2024-0001.json", + "modified": "2024-04-01T10:00:00Z" + }, + { + "id": "VMSA-2024-0002", + "url": "https://vmware.example/api/vmsa/VMSA-2024-0002.json", + "modified": "2024-04-02T09:00:00Z" + } +] diff --git a/src/StellaOps.Feedser.Source.Vndr.Vmware.Tests/Vmware/Fixtures/vmware-index-second.json b/src/StellaOps.Feedser.Source.Vndr.Vmware.Tests/Vmware/Fixtures/vmware-index-second.json new file mode 100644 index 00000000..a55e9df6 --- /dev/null +++ b/src/StellaOps.Feedser.Source.Vndr.Vmware.Tests/Vmware/Fixtures/vmware-index-second.json @@ -0,0 +1,17 @@ +[ + { + "id": "VMSA-2024-0001", + "url": "https://vmware.example/api/vmsa/VMSA-2024-0001.json", + "modified": "2024-04-01T10:00:00Z" + }, + { + "id": "VMSA-2024-0002", + "url": "https://vmware.example/api/vmsa/VMSA-2024-0002.json", + "modified": "2024-04-02T09:00:00Z" + }, + { + "id": "VMSA-2024-0003", + "url": "https://vmware.example/api/vmsa/VMSA-2024-0003.json", + "modified": "2024-04-03T08:15:00Z" + } +] diff --git a/src/StellaOps.Feedser.Source.Vndr.Vmware.Tests/Vmware/VmwareConnectorTests.cs b/src/StellaOps.Feedser.Source.Vndr.Vmware.Tests/Vmware/VmwareConnectorTests.cs new file mode 100644 index 00000000..a1f98a6c --- /dev/null +++ b/src/StellaOps.Feedser.Source.Vndr.Vmware.Tests/Vmware/VmwareConnectorTests.cs @@ -0,0 +1,266 @@ +using System; +using System.Collections.Concurrent; +using System.Collections.Generic; +using System.Diagnostics.Metrics; +using System.IO; +using System.Linq; +using System.Net.Http; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Http; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Logging.Abstractions; +using Microsoft.Extensions.Options; +using Microsoft.Extensions.Time.Testing; +using MongoDB.Bson; +using MongoDB.Driver; +using StellaOps.Feedser.Models; +using StellaOps.Feedser.Source.Common; +using StellaOps.Feedser.Source.Common.Http; +using StellaOps.Feedser.Source.Common.Testing; +using StellaOps.Feedser.Source.Vndr.Vmware; +using StellaOps.Feedser.Source.Vndr.Vmware.Configuration; +using StellaOps.Feedser.Source.Vndr.Vmware.Internal; +using StellaOps.Feedser.Storage.Mongo; +using StellaOps.Feedser.Storage.Mongo.Advisories; +using StellaOps.Feedser.Storage.Mongo.Documents; +using StellaOps.Feedser.Storage.Mongo.Dtos; +using StellaOps.Feedser.Testing; +using Xunit.Abstractions; + +namespace StellaOps.Feedser.Source.Vndr.Vmware.Tests.Vmware; + +[Collection("mongo-fixture")] +public sealed class VmwareConnectorTests : IAsyncLifetime +{ + private readonly MongoIntegrationFixture _fixture; + private readonly FakeTimeProvider _timeProvider; + private readonly CannedHttpMessageHandler _handler; + private readonly ITestOutputHelper _output; + + private static readonly Uri IndexUri = new("https://vmware.example/api/vmsa/index.json"); + private static readonly Uri DetailOne = new("https://vmware.example/api/vmsa/VMSA-2024-0001.json"); + private static readonly Uri DetailTwo = new("https://vmware.example/api/vmsa/VMSA-2024-0002.json"); + private static readonly Uri DetailThree = new("https://vmware.example/api/vmsa/VMSA-2024-0003.json"); + + public VmwareConnectorTests(MongoIntegrationFixture fixture, ITestOutputHelper output) + { + _fixture = fixture; + _timeProvider = new FakeTimeProvider(new DateTimeOffset(2024, 4, 5, 0, 0, 0, TimeSpan.Zero)); + _handler = new CannedHttpMessageHandler(); + _output = output; + } + + [Fact] + public async Task FetchParseMap_ProducesSnapshotAndCoversResume() + { + await using var provider = await BuildServiceProviderAsync(); + SeedInitialResponses(); + + using var metrics = new VmwareMetricCollector(); + + var connector = provider.GetRequiredService(); + + await connector.FetchAsync(provider, CancellationToken.None); + await connector.ParseAsync(provider, CancellationToken.None); + await connector.MapAsync(provider, CancellationToken.None); + + var advisoryStore = provider.GetRequiredService(); + var advisories = await advisoryStore.GetRecentAsync(10, CancellationToken.None); + var ordered = advisories.OrderBy(static a => a.AdvisoryKey, StringComparer.Ordinal).ToArray(); + + var snapshot = Normalize(SnapshotSerializer.ToSnapshot(ordered)); + var expected = Normalize(ReadFixture("vmware-advisories.snapshot.json")); + if (!string.Equals(expected, snapshot, StringComparison.Ordinal)) + { + var actualPath = Path.Combine(AppContext.BaseDirectory, "Vmware", "Fixtures", "vmware-advisories.actual.json"); + Directory.CreateDirectory(Path.GetDirectoryName(actualPath)!); + File.WriteAllText(actualPath, snapshot); + } + + Assert.Equal(expected, snapshot); + + var psirtCollection = _fixture.Database.GetCollection(MongoStorageDefaults.Collections.PsirtFlags); + var psirtFlags = await psirtCollection.Find(Builders.Filter.Empty).ToListAsync(); + _output.WriteLine("PSIRT flags after initial map: " + string.Join(", ", psirtFlags.Select(flag => flag.GetValue("_id", BsonValue.Create("")).ToString()))); + Assert.Equal(2, psirtFlags.Count); + Assert.All(psirtFlags, doc => Assert.Equal("VMware", doc["vendor"].AsString)); + + var stateRepository = provider.GetRequiredService(); + var state = await stateRepository.TryGetAsync(VmwareConnectorPlugin.SourceName, CancellationToken.None); + Assert.NotNull(state); + Assert.Empty(state!.Cursor.TryGetValue("pendingDocuments", out var pendingDocs) ? pendingDocs.AsBsonArray : new BsonArray()); + Assert.Empty(state.Cursor.TryGetValue("pendingMappings", out var pendingMaps) ? pendingMaps.AsBsonArray : new BsonArray()); + var cursorSnapshot = VmwareCursor.FromBson(state.Cursor); + _output.WriteLine($"Initial fetch cache entries: {cursorSnapshot.FetchCache.Count}"); + foreach (var entry in cursorSnapshot.FetchCache) + { + _output.WriteLine($"Cache seed: {entry.Key} -> {entry.Value.Sha256}"); + } + + // Second run with unchanged advisories and one new advisory. + SeedUpdateResponses(); + _timeProvider.Advance(TimeSpan.FromHours(1)); + + await connector.FetchAsync(provider, CancellationToken.None); + var documentStore = provider.GetRequiredService(); + var resumeDocOne = await documentStore.FindBySourceAndUriAsync(VmwareConnectorPlugin.SourceName, DetailOne.ToString(), CancellationToken.None); + var resumeDocTwo = await documentStore.FindBySourceAndUriAsync(VmwareConnectorPlugin.SourceName, DetailTwo.ToString(), CancellationToken.None); + _output.WriteLine($"After resume fetch status: {resumeDocOne?.Status} ({resumeDocOne?.Sha256}), {resumeDocTwo?.Status} ({resumeDocTwo?.Sha256})"); + Assert.Equal(DocumentStatuses.Mapped, resumeDocOne?.Status); + Assert.Equal(DocumentStatuses.Mapped, resumeDocTwo?.Status); + await connector.ParseAsync(provider, CancellationToken.None); + await connector.MapAsync(provider, CancellationToken.None); + + advisories = await advisoryStore.GetRecentAsync(10, CancellationToken.None); + Assert.Equal(3, advisories.Count); + Assert.Contains(advisories, advisory => advisory.AdvisoryKey == "VMSA-2024-0003"); + + psirtFlags = await psirtCollection.Find(Builders.Filter.Empty).ToListAsync(); + _output.WriteLine("PSIRT flags after resume: " + string.Join(", ", psirtFlags.Select(flag => flag.GetValue("_id", BsonValue.Create("")).ToString()))); + Assert.Equal(3, psirtFlags.Count); + Assert.Contains(psirtFlags, doc => doc["_id"] == "VMSA-2024-0003"); + + var measurements = metrics.Measurements; + _output.WriteLine("Captured metrics:"); + foreach (var measurement in measurements) + { + _output.WriteLine($"{measurement.Name} -> {measurement.Value}"); + } + + Assert.Equal(0, Sum(measurements, "vmware.fetch.failures")); + Assert.Equal(0, Sum(measurements, "vmware.parse.fail")); + Assert.Equal(3, Sum(measurements, "vmware.fetch.items")); // two initial, one new + + var affectedCounts = measurements + .Where(m => m.Name == "vmware.map.affected_count") + .Select(m => (int)m.Value) + .OrderBy(v => v) + .ToArray(); + Assert.Equal(new[] { 1, 1, 2 }, affectedCounts); + } + + public Task InitializeAsync() => Task.CompletedTask; + + public Task DisposeAsync() + { + _handler.Clear(); + return Task.CompletedTask; + } + + private async Task BuildServiceProviderAsync() + { + await _fixture.Client.DropDatabaseAsync(_fixture.Database.DatabaseNamespace.DatabaseName); + _handler.Clear(); + + var services = new ServiceCollection(); + services.AddLogging(builder => builder.AddProvider(NullLoggerProvider.Instance)); + services.AddSingleton(_timeProvider); + services.AddSingleton(_handler); + + services.AddMongoStorage(options => + { + options.ConnectionString = _fixture.Runner.ConnectionString; + options.DatabaseName = _fixture.Database.DatabaseNamespace.DatabaseName; + options.CommandTimeout = TimeSpan.FromSeconds(5); + }); + + services.AddSourceCommon(); + services.AddVmwareConnector(opts => + { + opts.IndexUri = IndexUri; + opts.InitialBackfill = TimeSpan.FromDays(30); + opts.ModifiedTolerance = TimeSpan.FromMinutes(5); + opts.MaxAdvisoriesPerFetch = 10; + opts.RequestDelay = TimeSpan.Zero; + }); + + services.Configure(VmwareOptions.HttpClientName, builderOptions => + { + builderOptions.HttpMessageHandlerBuilderActions.Add(builder => builder.PrimaryHandler = _handler); + }); + + var provider = services.BuildServiceProvider(); + var bootstrapper = provider.GetRequiredService(); + await bootstrapper.InitializeAsync(CancellationToken.None); + return provider; + } + + private void SeedInitialResponses() + { + _handler.AddJsonResponse(IndexUri, ReadFixture("vmware-index-initial.json")); + _handler.AddJsonResponse(DetailOne, ReadFixture("vmware-detail-vmsa-2024-0001.json")); + _handler.AddJsonResponse(DetailTwo, ReadFixture("vmware-detail-vmsa-2024-0002.json")); + } + + private void SeedUpdateResponses() + { + _handler.AddJsonResponse(IndexUri, ReadFixture("vmware-index-second.json")); + _handler.AddJsonResponse(DetailOne, ReadFixture("vmware-detail-vmsa-2024-0001.json")); + _handler.AddJsonResponse(DetailTwo, ReadFixture("vmware-detail-vmsa-2024-0002.json")); + _handler.AddJsonResponse(DetailThree, ReadFixture("vmware-detail-vmsa-2024-0003.json")); + } + + private static string ReadFixture(string name) + { + var primary = Path.Combine(AppContext.BaseDirectory, "Vmware", "Fixtures", name); + if (File.Exists(primary)) + { + return File.ReadAllText(primary); + } + + var fallback = Path.Combine(AppContext.BaseDirectory, "Fixtures", name); + if (File.Exists(fallback)) + { + return File.ReadAllText(fallback); + } + + throw new FileNotFoundException($"Fixture '{name}' not found.", name); + } + + private static string Normalize(string value) + => value.Replace("\r\n", "\n", StringComparison.Ordinal).TrimEnd(); + + private static long Sum(IEnumerable measurements, string name) + => measurements.Where(m => m.Name == name).Sum(m => m.Value); + + private sealed class VmwareMetricCollector : IDisposable + { + private readonly MeterListener _listener; + private readonly ConcurrentBag _measurements = new(); + + public VmwareMetricCollector() + { + _listener = new MeterListener + { + InstrumentPublished = (instrument, listener) => + { + if (instrument.Meter.Name == VmwareDiagnostics.MeterName) + { + listener.EnableMeasurementEvents(instrument); + } + } + }; + + _listener.SetMeasurementEventCallback((instrument, measurement, tags, state) => + { + var tagList = new List>(tags.Length); + foreach (var tag in tags) + { + tagList.Add(tag); + } + + _measurements.Add(new MetricMeasurement(instrument.Name, measurement, tagList)); + }); + + _listener.Start(); + } + + public IReadOnlyCollection Measurements => _measurements; + + public void Dispose() => _listener.Dispose(); + + public sealed record MetricMeasurement(string Name, long Value, IReadOnlyList> Tags); + } +} diff --git a/src/StellaOps.Feedser.Source.Vndr.Vmware.Tests/Vmware/VmwareMapperTests.cs b/src/StellaOps.Feedser.Source.Vndr.Vmware.Tests/Vmware/VmwareMapperTests.cs new file mode 100644 index 00000000..1f7c23b1 --- /dev/null +++ b/src/StellaOps.Feedser.Source.Vndr.Vmware.Tests/Vmware/VmwareMapperTests.cs @@ -0,0 +1,86 @@ +using System; +using System.Collections.Generic; +using System.Text.Json; +using MongoDB.Bson; +using StellaOps.Feedser.Models; +using StellaOps.Feedser.Source.Common; +using StellaOps.Feedser.Source.Vndr.Vmware; +using StellaOps.Feedser.Source.Vndr.Vmware.Internal; +using StellaOps.Feedser.Storage.Mongo.Documents; +using StellaOps.Feedser.Storage.Mongo.Dtos; +using Xunit; + +namespace StellaOps.Feedser.Source.Vndr.Vmware.Tests; + +public sealed class VmwareMapperTests +{ + [Fact] + public void Map_CreatesCanonicalAdvisory() + { + var modified = DateTimeOffset.UtcNow; + var dto = new VmwareDetailDto + { + AdvisoryId = "VMSA-2025-0001", + Title = "Sample VMware Advisory", + Summary = "Summary text", + Published = modified.AddDays(-1), + Modified = modified, + CveIds = new[] { "CVE-2025-0001", "CVE-2025-0002" }, + References = new[] + { + new VmwareReferenceDto { Url = "https://kb.vmware.com/some-kb", Type = "KB" }, + new VmwareReferenceDto { Url = "https://vmsa.vmware.com/vmsa/KB", Type = "Advisory" }, + }, + Affected = new[] + { + new VmwareAffectedProductDto + { + Product = "VMware vCenter", + Version = "7.0", + FixedVersion = "7.0u3" + } + } + }; + + var document = new DocumentRecord( + Guid.NewGuid(), + VmwareConnectorPlugin.SourceName, + "https://vmsa.vmware.com/vmsa/VMSA-2025-0001", + DateTimeOffset.UtcNow, + "sha256", + DocumentStatuses.PendingParse, + "application/json", + null, + new Dictionary(StringComparer.Ordinal) + { + ["vmware.id"] = dto.AdvisoryId, + }, + null, + modified, + null, + null); + + var payload = BsonDocument.Parse(JsonSerializer.Serialize(dto, new JsonSerializerOptions(JsonSerializerDefaults.Web) + { + DefaultIgnoreCondition = System.Text.Json.Serialization.JsonIgnoreCondition.WhenWritingNull, + })); + + var dtoRecord = new DtoRecord(Guid.NewGuid(), document.Id, VmwareConnectorPlugin.SourceName, "vmware.v1", payload, DateTimeOffset.UtcNow); + + var (advisory, flag) = VmwareMapper.Map(dto, document, dtoRecord); + + Assert.Equal(dto.AdvisoryId, advisory.AdvisoryKey); + Assert.Contains("CVE-2025-0001", advisory.Aliases); + Assert.Contains("CVE-2025-0002", advisory.Aliases); + Assert.Single(advisory.AffectedPackages); + Assert.Equal("VMware vCenter", advisory.AffectedPackages[0].Identifier); + Assert.Single(advisory.AffectedPackages[0].VersionRanges); + Assert.Equal("7.0", advisory.AffectedPackages[0].VersionRanges[0].IntroducedVersion); + Assert.Equal("7.0u3", advisory.AffectedPackages[0].VersionRanges[0].FixedVersion); + Assert.Equal(2, advisory.References.Length); + Assert.Equal("https://kb.vmware.com/some-kb", advisory.References[0].Url); + Assert.Equal(dto.AdvisoryId, flag.AdvisoryKey); + Assert.Equal("VMware", flag.Vendor); + Assert.Equal(VmwareConnectorPlugin.SourceName, flag.SourceName); + } +} diff --git a/src/StellaOps.Feedser.Source.Vndr.Vmware/AGENTS.md b/src/StellaOps.Feedser.Source.Vndr.Vmware/AGENTS.md new file mode 100644 index 00000000..724c6e4e --- /dev/null +++ b/src/StellaOps.Feedser.Source.Vndr.Vmware/AGENTS.md @@ -0,0 +1,28 @@ +# AGENTS +## Role +VMware/Broadcom PSIRT connector ingesting VMSA advisories; authoritative for VMware products; maps affected versions/builds and emits psirt_flags. +## Scope +- Discover/fetch VMSA index and detail pages via Broadcom portal; window by advisory ID/date; follow updates/revisions. +- Validate HTML or JSON; extract CVEs, affected product versions/builds, workarounds, fixed versions; normalize product naming. +- Persist raw docs with sha256; manage source_state; idempotent mapping. +## Participants +- Source.Common (HTTP, cookies/session handling if needed, validators). +- Storage.Mongo (document, dto, advisory, alias, affected, reference, psirt_flags, source_state). +- Models (canonical). +- Core/WebService (jobs: source:vmware:fetch|parse|map). +- Merge engine (later) to prefer PSIRT ranges for VMware products. +## Interfaces & contracts +- Aliases: VMSA-YYYY-NNNN plus CVEs. +- Affected entries include Vendor=VMware, Product plus component; Versions carry fixed/fixedBy; tags may include build numbers or ESXi/VC levels. +- References: advisory URL, KBs, workaround pages; typed; deduped. +- Provenance: method=parser; value=VMSA id. +## In/Out of scope +In: PSIRT precedence mapping, affected/fixedBy extraction, advisory references. +Out: customer portal authentication flows beyond public advisories; downloading patches. +## Observability & security expectations +- Metrics: SourceDiagnostics emits shared `feedser.source.http.*` counters/histograms tagged `feedser.source=vmware`, allowing dashboards to measure fetch volume, parse failures, and map affected counts without bespoke metric names. +- Logs: vmsa ids, product counts, extraction timings; handle portal rate limits politely. +## Tests +- Author and review coverage in `../StellaOps.Feedser.Source.Vndr.Vmware.Tests`. +- Shared fixtures (e.g., `MongoIntegrationFixture`, `ConnectorTestHarness`) live in `../StellaOps.Feedser.Testing`. +- Keep fixtures deterministic; match new cases to real-world advisories or regression scenarios. diff --git a/src/StellaOps.Feedser.Source.Vndr.Vmware/Configuration/VmwareOptions.cs b/src/StellaOps.Feedser.Source.Vndr.Vmware/Configuration/VmwareOptions.cs new file mode 100644 index 00000000..362c07c8 --- /dev/null +++ b/src/StellaOps.Feedser.Source.Vndr.Vmware/Configuration/VmwareOptions.cs @@ -0,0 +1,54 @@ +using System.Diagnostics.CodeAnalysis; + +namespace StellaOps.Feedser.Source.Vndr.Vmware.Configuration; + +public sealed class VmwareOptions +{ + public const string HttpClientName = "source.vmware"; + + public Uri IndexUri { get; set; } = new("https://example.invalid/vmsa/index.json", UriKind.Absolute); + + public TimeSpan InitialBackfill { get; set; } = TimeSpan.FromDays(30); + + public TimeSpan ModifiedTolerance { get; set; } = TimeSpan.FromHours(2); + + public int MaxAdvisoriesPerFetch { get; set; } = 50; + + public TimeSpan RequestDelay { get; set; } = TimeSpan.FromMilliseconds(250); + + public TimeSpan HttpTimeout { get; set; } = TimeSpan.FromMinutes(2); + + [MemberNotNull(nameof(IndexUri))] + public void Validate() + { + if (IndexUri is null || !IndexUri.IsAbsoluteUri) + { + throw new InvalidOperationException("VMware index URI must be absolute."); + } + + if (InitialBackfill <= TimeSpan.Zero) + { + throw new InvalidOperationException("Initial backfill must be positive."); + } + + if (ModifiedTolerance < TimeSpan.Zero) + { + throw new InvalidOperationException("Modified tolerance cannot be negative."); + } + + if (MaxAdvisoriesPerFetch <= 0) + { + throw new InvalidOperationException("Max advisories per fetch must be greater than zero."); + } + + if (RequestDelay < TimeSpan.Zero) + { + throw new InvalidOperationException("Request delay cannot be negative."); + } + + if (HttpTimeout <= TimeSpan.Zero) + { + throw new InvalidOperationException("HTTP timeout must be positive."); + } + } +} diff --git a/src/StellaOps.Feedser.Source.Vndr.Vmware/Internal/VmwareCursor.cs b/src/StellaOps.Feedser.Source.Vndr.Vmware/Internal/VmwareCursor.cs new file mode 100644 index 00000000..44b27ca8 --- /dev/null +++ b/src/StellaOps.Feedser.Source.Vndr.Vmware/Internal/VmwareCursor.cs @@ -0,0 +1,172 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using MongoDB.Bson; + +namespace StellaOps.Feedser.Source.Vndr.Vmware.Internal; + +internal sealed record VmwareCursor( + DateTimeOffset? LastModified, + IReadOnlyCollection ProcessedIds, + IReadOnlyCollection PendingDocuments, + IReadOnlyCollection PendingMappings, + IReadOnlyDictionary FetchCache) +{ + private static readonly IReadOnlyCollection EmptyGuidList = Array.Empty(); + private static readonly IReadOnlyCollection EmptyStringList = Array.Empty(); + private static readonly IReadOnlyDictionary EmptyFetchCache = + new Dictionary(StringComparer.OrdinalIgnoreCase); + + public static VmwareCursor Empty { get; } = new(null, EmptyStringList, EmptyGuidList, EmptyGuidList, EmptyFetchCache); + + public BsonDocument ToBsonDocument() + { + var document = new BsonDocument + { + ["pendingDocuments"] = new BsonArray(PendingDocuments.Select(id => id.ToString())), + ["pendingMappings"] = new BsonArray(PendingMappings.Select(id => id.ToString())), + }; + + if (LastModified.HasValue) + { + document["lastModified"] = LastModified.Value.UtcDateTime; + } + + if (ProcessedIds.Count > 0) + { + document["processedIds"] = new BsonArray(ProcessedIds); + } + + if (FetchCache.Count > 0) + { + var cacheDocument = new BsonDocument(); + foreach (var (key, entry) in FetchCache) + { + cacheDocument[key] = entry.ToBsonDocument(); + } + + document["fetchCache"] = cacheDocument; + } + + return document; + } + + public static VmwareCursor FromBson(BsonDocument? document) + { + if (document is null || document.ElementCount == 0) + { + return Empty; + } + + var lastModified = document.TryGetValue("lastModified", out var value) + ? ParseDate(value) + : null; + + var processedIds = document.TryGetValue("processedIds", out var processedValue) && processedValue is BsonArray idsArray + ? idsArray.OfType() + .Where(static x => x.BsonType == BsonType.String) + .Select(static x => x.AsString) + .ToArray() + : EmptyStringList; + + var pendingDocuments = ReadGuidArray(document, "pendingDocuments"); + var pendingMappings = ReadGuidArray(document, "pendingMappings"); + var fetchCache = ReadFetchCache(document); + + return new VmwareCursor(lastModified, processedIds, pendingDocuments, pendingMappings, fetchCache); + } + + public VmwareCursor WithLastModified(DateTimeOffset timestamp, IEnumerable processedIds) + => this with + { + LastModified = timestamp.ToUniversalTime(), + ProcessedIds = processedIds?.Where(static id => !string.IsNullOrWhiteSpace(id)) + .Select(static id => id.Trim()) + .Distinct(StringComparer.OrdinalIgnoreCase) + .ToArray() ?? EmptyStringList, + }; + + public VmwareCursor WithPendingDocuments(IEnumerable ids) + => this with { PendingDocuments = ids?.Distinct().ToArray() ?? EmptyGuidList }; + + public VmwareCursor WithPendingMappings(IEnumerable ids) + => this with { PendingMappings = ids?.Distinct().ToArray() ?? EmptyGuidList }; + + public VmwareCursor WithFetchCache(IDictionary? cache) + { + if (cache is null || cache.Count == 0) + { + return this with { FetchCache = EmptyFetchCache }; + } + + return this with { FetchCache = new Dictionary(cache, StringComparer.OrdinalIgnoreCase) }; + } + + public bool TryGetFetchCache(string key, out VmwareFetchCacheEntry entry) + { + if (FetchCache.Count == 0) + { + entry = VmwareFetchCacheEntry.Empty; + return false; + } + + return FetchCache.TryGetValue(key, out entry!); + } + + public VmwareCursor AddProcessedId(string id) + { + if (string.IsNullOrWhiteSpace(id)) + { + return this; + } + + var set = new HashSet(ProcessedIds, StringComparer.OrdinalIgnoreCase) { id.Trim() }; + return this with { ProcessedIds = set.ToArray() }; + } + + private static IReadOnlyCollection ReadGuidArray(BsonDocument document, string field) + { + if (!document.TryGetValue(field, out var value) || value is not BsonArray array) + { + return EmptyGuidList; + } + + var results = new List(array.Count); + foreach (var element in array) + { + if (Guid.TryParse(element.ToString(), out var guid)) + { + results.Add(guid); + } + } + + return results; + } + + private static IReadOnlyDictionary ReadFetchCache(BsonDocument document) + { + if (!document.TryGetValue("fetchCache", out var value) || value is not BsonDocument cacheDocument || cacheDocument.ElementCount == 0) + { + return EmptyFetchCache; + } + + var cache = new Dictionary(StringComparer.OrdinalIgnoreCase); + foreach (var element in cacheDocument.Elements) + { + if (element.Value is BsonDocument entryDocument) + { + cache[element.Name] = VmwareFetchCacheEntry.FromBson(entryDocument); + } + } + + return cache; + } + + private static DateTimeOffset? ParseDate(BsonValue value) + => value.BsonType switch + { + BsonType.DateTime => DateTime.SpecifyKind(value.ToUniversalTime(), DateTimeKind.Utc), + BsonType.String when DateTimeOffset.TryParse(value.AsString, out var parsed) => parsed.ToUniversalTime(), + _ => null, + }; +} diff --git a/src/StellaOps.Feedser.Source.Vndr.Vmware/Internal/VmwareDetailDto.cs b/src/StellaOps.Feedser.Source.Vndr.Vmware/Internal/VmwareDetailDto.cs new file mode 100644 index 00000000..d8d54799 --- /dev/null +++ b/src/StellaOps.Feedser.Source.Vndr.Vmware/Internal/VmwareDetailDto.cs @@ -0,0 +1,53 @@ +using System; +using System.Collections.Generic; +using System.Text.Json.Serialization; + +namespace StellaOps.Feedser.Source.Vndr.Vmware.Internal; + +internal sealed record VmwareDetailDto +{ + [JsonPropertyName("id")] + public string AdvisoryId { get; init; } = string.Empty; + + [JsonPropertyName("title")] + public string Title { get; init; } = string.Empty; + + [JsonPropertyName("summary")] + public string? Summary { get; init; } + + [JsonPropertyName("published")] + public DateTimeOffset? Published { get; init; } + + [JsonPropertyName("modified")] + public DateTimeOffset? Modified { get; init; } + + [JsonPropertyName("cves")] + public IReadOnlyList? CveIds { get; init; } + + [JsonPropertyName("affected")] + public IReadOnlyList? Affected { get; init; } + + [JsonPropertyName("references")] + public IReadOnlyList? References { get; init; } +} + +internal sealed record VmwareAffectedProductDto +{ + [JsonPropertyName("product")] + public string Product { get; init; } = string.Empty; + + [JsonPropertyName("version")] + public string? Version { get; init; } + + [JsonPropertyName("fixedVersion")] + public string? FixedVersion { get; init; } +} + +internal sealed record VmwareReferenceDto +{ + [JsonPropertyName("type")] + public string? Type { get; init; } + + [JsonPropertyName("url")] + public string Url { get; init; } = string.Empty; +} diff --git a/src/StellaOps.Feedser.Source.Vndr.Vmware/Internal/VmwareFetchCacheEntry.cs b/src/StellaOps.Feedser.Source.Vndr.Vmware/Internal/VmwareFetchCacheEntry.cs new file mode 100644 index 00000000..89634a6d --- /dev/null +++ b/src/StellaOps.Feedser.Source.Vndr.Vmware/Internal/VmwareFetchCacheEntry.cs @@ -0,0 +1,88 @@ +using System; +using MongoDB.Bson; +using StellaOps.Feedser.Storage.Mongo.Documents; + +namespace StellaOps.Feedser.Source.Vndr.Vmware.Internal; + +internal sealed record VmwareFetchCacheEntry(string? Sha256, string? ETag, DateTimeOffset? LastModified) +{ + public static VmwareFetchCacheEntry Empty { get; } = new(string.Empty, null, null); + + public BsonDocument ToBsonDocument() + { + var document = new BsonDocument + { + ["sha256"] = Sha256 ?? string.Empty, + }; + + if (!string.IsNullOrWhiteSpace(ETag)) + { + document["etag"] = ETag; + } + + if (LastModified.HasValue) + { + document["lastModified"] = LastModified.Value.UtcDateTime; + } + + return document; + } + + public static VmwareFetchCacheEntry FromBson(BsonDocument document) + { + var sha256 = document.TryGetValue("sha256", out var shaValue) ? shaValue.ToString() : string.Empty; + string? etag = null; + if (document.TryGetValue("etag", out var etagValue) && !etagValue.IsBsonNull) + { + etag = etagValue.ToString(); + } + + DateTimeOffset? lastModified = null; + if (document.TryGetValue("lastModified", out var lastModifiedValue)) + { + lastModified = lastModifiedValue.BsonType switch + { + BsonType.DateTime => DateTime.SpecifyKind(lastModifiedValue.ToUniversalTime(), DateTimeKind.Utc), + BsonType.String when DateTimeOffset.TryParse(lastModifiedValue.AsString, out var parsed) => parsed.ToUniversalTime(), + _ => null, + }; + } + + return new VmwareFetchCacheEntry(sha256, etag, lastModified); + } + + public static VmwareFetchCacheEntry FromDocument(DocumentRecord document) + { + ArgumentNullException.ThrowIfNull(document); + + return new VmwareFetchCacheEntry( + document.Sha256, + document.Etag, + document.LastModified?.ToUniversalTime()); + } + + public bool Matches(DocumentRecord document) + { + ArgumentNullException.ThrowIfNull(document); + + if (!string.IsNullOrEmpty(Sha256) && !string.IsNullOrEmpty(document.Sha256) + && string.Equals(Sha256, document.Sha256, StringComparison.OrdinalIgnoreCase)) + { + return true; + } + + if (!string.IsNullOrEmpty(ETag) && !string.IsNullOrEmpty(document.Etag) + && string.Equals(ETag, document.Etag, StringComparison.Ordinal)) + { + return true; + } + + if (LastModified.HasValue && document.LastModified.HasValue + && LastModified.Value.ToUniversalTime() == document.LastModified.Value.ToUniversalTime()) + { + return true; + } + + return false; + } +} diff --git a/src/StellaOps.Feedser.Source.Vndr.Vmware/Internal/VmwareIndexItem.cs b/src/StellaOps.Feedser.Source.Vndr.Vmware/Internal/VmwareIndexItem.cs new file mode 100644 index 00000000..2099cdb4 --- /dev/null +++ b/src/StellaOps.Feedser.Source.Vndr.Vmware/Internal/VmwareIndexItem.cs @@ -0,0 +1,16 @@ +using System; +using System.Text.Json.Serialization; + +namespace StellaOps.Feedser.Source.Vndr.Vmware.Internal; + +internal sealed record VmwareIndexItem +{ + [JsonPropertyName("id")] + public string Id { get; init; } = string.Empty; + + [JsonPropertyName("url")] + public string DetailUrl { get; init; } = string.Empty; + + [JsonPropertyName("modified")] + public DateTimeOffset? Modified { get; init; } +} diff --git a/src/StellaOps.Feedser.Source.Vndr.Vmware/Internal/VmwareMapper.cs b/src/StellaOps.Feedser.Source.Vndr.Vmware/Internal/VmwareMapper.cs new file mode 100644 index 00000000..9602719f --- /dev/null +++ b/src/StellaOps.Feedser.Source.Vndr.Vmware/Internal/VmwareMapper.cs @@ -0,0 +1,235 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using StellaOps.Feedser.Models; +using StellaOps.Feedser.Source.Common; +using StellaOps.Feedser.Source.Common.Packages; +using StellaOps.Feedser.Storage.Mongo.Documents; +using StellaOps.Feedser.Storage.Mongo.Dtos; +using StellaOps.Feedser.Storage.Mongo.PsirtFlags; + +namespace StellaOps.Feedser.Source.Vndr.Vmware.Internal; + +internal static class VmwareMapper +{ + public static (Advisory Advisory, PsirtFlagRecord Flag) Map(VmwareDetailDto dto, DocumentRecord document, DtoRecord dtoRecord) + { + ArgumentNullException.ThrowIfNull(dto); + ArgumentNullException.ThrowIfNull(document); + ArgumentNullException.ThrowIfNull(dtoRecord); + + var recordedAt = dtoRecord.ValidatedAt.ToUniversalTime(); + var fetchProvenance = new AdvisoryProvenance(VmwareConnectorPlugin.SourceName, "document", document.Uri, document.FetchedAt.ToUniversalTime()); + var mappingProvenance = new AdvisoryProvenance(VmwareConnectorPlugin.SourceName, "mapping", dto.AdvisoryId, recordedAt); + + var aliases = BuildAliases(dto); + var references = BuildReferences(dto, recordedAt); + var affectedPackages = BuildAffectedPackages(dto, recordedAt); + + var advisory = new Advisory( + dto.AdvisoryId, + dto.Title, + dto.Summary, + language: "en", + dto.Published?.ToUniversalTime(), + dto.Modified?.ToUniversalTime(), + severity: null, + exploitKnown: false, + aliases, + references, + affectedPackages, + cvssMetrics: Array.Empty(), + provenance: new[] { fetchProvenance, mappingProvenance }); + + var flag = new PsirtFlagRecord( + dto.AdvisoryId, + "VMware", + VmwareConnectorPlugin.SourceName, + dto.AdvisoryId, + recordedAt); + + return (advisory, flag); + } + + private static IEnumerable BuildAliases(VmwareDetailDto dto) + { + var set = new HashSet(StringComparer.OrdinalIgnoreCase) { dto.AdvisoryId }; + if (dto.CveIds is not null) + { + foreach (var cve in dto.CveIds) + { + if (!string.IsNullOrWhiteSpace(cve)) + { + set.Add(cve.Trim()); + } + } + } + + return set; + } + + private static IReadOnlyList BuildReferences(VmwareDetailDto dto, DateTimeOffset recordedAt) + { + if (dto.References is null || dto.References.Count == 0) + { + return Array.Empty(); + } + + var references = new List(dto.References.Count); + foreach (var reference in dto.References) + { + if (string.IsNullOrWhiteSpace(reference.Url)) + { + continue; + } + + var kind = NormalizeReferenceKind(reference.Type); + var provenance = new AdvisoryProvenance(VmwareConnectorPlugin.SourceName, "reference", reference.Url, recordedAt); + try + { + references.Add(new AdvisoryReference(reference.Url, kind, reference.Type, null, provenance)); + } + catch (ArgumentException) + { + // ignore invalid urls + } + } + + references.Sort(static (left, right) => StringComparer.OrdinalIgnoreCase.Compare(left.Url, right.Url)); + return references.Count == 0 ? Array.Empty() : references; + } + + private static string? NormalizeReferenceKind(string? type) + { + if (string.IsNullOrWhiteSpace(type)) + { + return null; + } + + return type.Trim().ToLowerInvariant() switch + { + "advisory" => "advisory", + "kb" or "kb_article" => "kb", + "patch" => "patch", + "workaround" => "workaround", + _ => null, + }; + } + + private static IReadOnlyList BuildAffectedPackages(VmwareDetailDto dto, DateTimeOffset recordedAt) + { + if (dto.Affected is null || dto.Affected.Count == 0) + { + return Array.Empty(); + } + + var packages = new List(dto.Affected.Count); + foreach (var product in dto.Affected) + { + if (string.IsNullOrWhiteSpace(product.Product)) + { + continue; + } + + var provenance = new[] + { + new AdvisoryProvenance(VmwareConnectorPlugin.SourceName, "affected", product.Product, recordedAt), + }; + + var ranges = new List(); + if (!string.IsNullOrWhiteSpace(product.Version) || !string.IsNullOrWhiteSpace(product.FixedVersion)) + { + var rangeProvenance = new AdvisoryProvenance(VmwareConnectorPlugin.SourceName, "range", product.Product, recordedAt); + ranges.Add(new AffectedVersionRange( + rangeKind: "vendor", + introducedVersion: product.Version, + fixedVersion: product.FixedVersion, + lastAffectedVersion: null, + rangeExpression: product.Version, + provenance: rangeProvenance, + primitives: BuildRangePrimitives(product))); + } + + packages.Add(new AffectedPackage( + AffectedPackageTypes.Vendor, + product.Product, + platform: null, + versionRanges: ranges, + statuses: Array.Empty(), + provenance: provenance)); + } + + return packages; + } + + private static RangePrimitives? BuildRangePrimitives(VmwareAffectedProductDto product) + { + var extensions = new Dictionary(StringComparer.Ordinal); + AddExtension(extensions, "vmware.product", product.Product); + AddExtension(extensions, "vmware.version.raw", product.Version); + AddExtension(extensions, "vmware.fixedVersion.raw", product.FixedVersion); + + var semVer = BuildSemVerPrimitive(product.Version, product.FixedVersion); + if (semVer is null && extensions.Count == 0) + { + return null; + } + + return new RangePrimitives(semVer, null, null, extensions.Count == 0 ? null : extensions); + } + + private static SemVerPrimitive? BuildSemVerPrimitive(string? introduced, string? fixedVersion) + { + var introducedNormalized = NormalizeSemVer(introduced); + var fixedNormalized = NormalizeSemVer(fixedVersion); + + if (introducedNormalized is null && fixedNormalized is null) + { + return null; + } + + return new SemVerPrimitive( + introducedNormalized, + IntroducedInclusive: true, + fixedNormalized, + FixedInclusive: false, + LastAffected: null, + LastAffectedInclusive: false, + ConstraintExpression: null); + } + + private static string? NormalizeSemVer(string? value) + { + if (PackageCoordinateHelper.TryParseSemVer(value, out _, out var normalized) && !string.IsNullOrWhiteSpace(normalized)) + { + return normalized; + } + + if (Version.TryParse(value, out var parsed)) + { + if (parsed.Build >= 0 && parsed.Revision >= 0) + { + return $"{parsed.Major}.{parsed.Minor}.{parsed.Build}.{parsed.Revision}"; + } + + if (parsed.Build >= 0) + { + return $"{parsed.Major}.{parsed.Minor}.{parsed.Build}"; + } + + return $"{parsed.Major}.{parsed.Minor}"; + } + + return null; + } + + private static void AddExtension(Dictionary extensions, string key, string? value) + { + if (string.IsNullOrWhiteSpace(value)) + { + return; + } + + extensions[key] = value.Trim(); + } +} diff --git a/src/StellaOps.Feedser.Source.Vndr.Vmware/Jobs.cs b/src/StellaOps.Feedser.Source.Vndr.Vmware/Jobs.cs new file mode 100644 index 00000000..14ebcec8 --- /dev/null +++ b/src/StellaOps.Feedser.Source.Vndr.Vmware/Jobs.cs @@ -0,0 +1,46 @@ +using System; +using System.Threading; +using System.Threading.Tasks; +using StellaOps.Feedser.Core.Jobs; + +namespace StellaOps.Feedser.Source.Vndr.Vmware; + +internal static class VmwareJobKinds +{ + public const string Fetch = "source:vmware:fetch"; + public const string Parse = "source:vmware:parse"; + public const string Map = "source:vmware:map"; +} + +internal sealed class VmwareFetchJob : IJob +{ + private readonly VmwareConnector _connector; + + public VmwareFetchJob(VmwareConnector connector) + => _connector = connector ?? throw new ArgumentNullException(nameof(connector)); + + public Task ExecuteAsync(JobExecutionContext context, CancellationToken cancellationToken) + => _connector.FetchAsync(context.Services, cancellationToken); +} + +internal sealed class VmwareParseJob : IJob +{ + private readonly VmwareConnector _connector; + + public VmwareParseJob(VmwareConnector connector) + => _connector = connector ?? throw new ArgumentNullException(nameof(connector)); + + public Task ExecuteAsync(JobExecutionContext context, CancellationToken cancellationToken) + => _connector.ParseAsync(context.Services, cancellationToken); +} + +internal sealed class VmwareMapJob : IJob +{ + private readonly VmwareConnector _connector; + + public VmwareMapJob(VmwareConnector connector) + => _connector = connector ?? throw new ArgumentNullException(nameof(connector)); + + public Task ExecuteAsync(JobExecutionContext context, CancellationToken cancellationToken) + => _connector.MapAsync(context.Services, cancellationToken); +} diff --git a/src/StellaOps.Feedser.Source.Vndr.Vmware/Properties/AssemblyInfo.cs b/src/StellaOps.Feedser.Source.Vndr.Vmware/Properties/AssemblyInfo.cs new file mode 100644 index 00000000..1e127068 --- /dev/null +++ b/src/StellaOps.Feedser.Source.Vndr.Vmware/Properties/AssemblyInfo.cs @@ -0,0 +1,3 @@ +using System.Runtime.CompilerServices; + +[assembly: InternalsVisibleTo("StellaOps.Feedser.Source.Vndr.Vmware.Tests")] diff --git a/src/StellaOps.Feedser.Source.Vndr.Vmware/StellaOps.Feedser.Source.Vndr.Vmware.csproj b/src/StellaOps.Feedser.Source.Vndr.Vmware/StellaOps.Feedser.Source.Vndr.Vmware.csproj new file mode 100644 index 00000000..76cc57c2 --- /dev/null +++ b/src/StellaOps.Feedser.Source.Vndr.Vmware/StellaOps.Feedser.Source.Vndr.Vmware.csproj @@ -0,0 +1,23 @@ + + + + net10.0 + enable + enable + + + + + + + + + + + + + <_Parameter1>StellaOps.Feedser.Tests + + + + diff --git a/src/StellaOps.Feedser.Source.Vndr.Vmware/TASKS.md b/src/StellaOps.Feedser.Source.Vndr.Vmware/TASKS.md new file mode 100644 index 00000000..3215f711 --- /dev/null +++ b/src/StellaOps.Feedser.Source.Vndr.Vmware/TASKS.md @@ -0,0 +1,17 @@ +# Source.Vndr.Vmware — Task Board + +| ID | Task | Owner | Status | Depends On | Notes | +|------|-----------------------------------------------|-------|--------|------------|-------| +| VM1 | Advisory listing discovery + cursor | Conn | DONE | Common | **DONE** – fetch pipeline uses index JSON with sliding cursor + processed id tracking. | +| VM2 | VMSA parser → DTO | QA | DONE | | **DONE** – JSON DTO deserialization wired with sanitization. | +| VM3 | Canonical mapping (aliases/affected/refs) | Conn | DONE | Models | **DONE** – `VmwareMapper` emits aliases/affected/reference ordering and persists PSIRT flags via `PsirtFlagStore`. | +| VM4 | Snapshot tests + resume | QA | DONE | Storage | **DONE** – integration test validates snapshot output and resume flow with cached state. | +| VM5 | Observability | QA | DONE | | **DONE** – diagnostics meter exposes fetch/parse/map metrics and structured logs. | +| VM6 | SourceState + hash dedupe | Conn | DONE | Storage | **DONE** – fetch cache stores sha/etag to skip unchanged advisories during resume. | +| VM6a | Options & HttpClient configuration | Conn | DONE | Source.Common | **DONE** – `AddVmwareConnector` configures allowlisted HttpClient + options. | +| VM7 | Dependency injection routine & scheduler registration | Conn | DONE | Core | **DONE** – `VmwareDependencyInjectionRoutine` registers fetch/parse/map jobs. | +| VM8 | Replace stub plugin with connector pipeline skeleton | Conn | DONE | Source.Common | **DONE** – connector implements fetch/parse/map persisting docs, DTOs, advisories. | +| VM9 | Range primitives + provenance diagnostics refresh | Conn | DONE | Models, Storage.Mongo | Vendor primitives emitted (SemVer + vendor extensions), provenance tags/logging updated, snapshots refreshed. | + +## Changelog +- YYYY-MM-DD: Created. diff --git a/src/StellaOps.Feedser.Source.Vndr.Vmware/VmwareConnector.cs b/src/StellaOps.Feedser.Source.Vndr.Vmware/VmwareConnector.cs new file mode 100644 index 00000000..f5fe1c3a --- /dev/null +++ b/src/StellaOps.Feedser.Source.Vndr.Vmware/VmwareConnector.cs @@ -0,0 +1,454 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Net.Http; +using System.Text.Json; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Options; +using MongoDB.Bson; +using MongoDB.Bson.IO; +using StellaOps.Feedser.Models; +using StellaOps.Feedser.Source.Common; +using StellaOps.Feedser.Source.Common.Fetch; +using StellaOps.Feedser.Source.Vndr.Vmware.Configuration; +using StellaOps.Feedser.Source.Vndr.Vmware.Internal; +using StellaOps.Feedser.Storage.Mongo; +using StellaOps.Feedser.Storage.Mongo.Advisories; +using StellaOps.Feedser.Storage.Mongo.Documents; +using StellaOps.Feedser.Storage.Mongo.Dtos; +using StellaOps.Feedser.Storage.Mongo.PsirtFlags; +using StellaOps.Plugin; + +namespace StellaOps.Feedser.Source.Vndr.Vmware; + +public sealed class VmwareConnector : IFeedConnector +{ + private static readonly JsonSerializerOptions SerializerOptions = new(JsonSerializerDefaults.Web) + { + PropertyNameCaseInsensitive = true, + DefaultIgnoreCondition = System.Text.Json.Serialization.JsonIgnoreCondition.WhenWritingNull, + }; + + private readonly IHttpClientFactory _httpClientFactory; + private readonly SourceFetchService _fetchService; + private readonly RawDocumentStorage _rawDocumentStorage; + private readonly IDocumentStore _documentStore; + private readonly IDtoStore _dtoStore; + private readonly IAdvisoryStore _advisoryStore; + private readonly ISourceStateRepository _stateRepository; + private readonly IPsirtFlagStore _psirtFlagStore; + private readonly VmwareOptions _options; + private readonly TimeProvider _timeProvider; + private readonly VmwareDiagnostics _diagnostics; + private readonly ILogger _logger; + + public VmwareConnector( + IHttpClientFactory httpClientFactory, + SourceFetchService fetchService, + RawDocumentStorage rawDocumentStorage, + IDocumentStore documentStore, + IDtoStore dtoStore, + IAdvisoryStore advisoryStore, + ISourceStateRepository stateRepository, + IPsirtFlagStore psirtFlagStore, + IOptions options, + TimeProvider? timeProvider, + VmwareDiagnostics diagnostics, + ILogger logger) + { + _httpClientFactory = httpClientFactory ?? throw new ArgumentNullException(nameof(httpClientFactory)); + _fetchService = fetchService ?? throw new ArgumentNullException(nameof(fetchService)); + _rawDocumentStorage = rawDocumentStorage ?? throw new ArgumentNullException(nameof(rawDocumentStorage)); + _documentStore = documentStore ?? throw new ArgumentNullException(nameof(documentStore)); + _dtoStore = dtoStore ?? throw new ArgumentNullException(nameof(dtoStore)); + _advisoryStore = advisoryStore ?? throw new ArgumentNullException(nameof(advisoryStore)); + _stateRepository = stateRepository ?? throw new ArgumentNullException(nameof(stateRepository)); + _psirtFlagStore = psirtFlagStore ?? throw new ArgumentNullException(nameof(psirtFlagStore)); + _options = (options ?? throw new ArgumentNullException(nameof(options))).Value ?? throw new ArgumentNullException(nameof(options)); + _options.Validate(); + _timeProvider = timeProvider ?? TimeProvider.System; + _diagnostics = diagnostics ?? throw new ArgumentNullException(nameof(diagnostics)); + _logger = logger ?? throw new ArgumentNullException(nameof(logger)); + } + + public string SourceName => VmwareConnectorPlugin.SourceName; + + public async Task FetchAsync(IServiceProvider services, CancellationToken cancellationToken) + { + ArgumentNullException.ThrowIfNull(services); + + var cursor = await GetCursorAsync(cancellationToken).ConfigureAwait(false); + var now = _timeProvider.GetUtcNow(); + var pendingDocuments = cursor.PendingDocuments.ToHashSet(); + var pendingMappings = cursor.PendingMappings.ToHashSet(); + var fetchCache = new Dictionary(cursor.FetchCache, StringComparer.OrdinalIgnoreCase); + var touchedResources = new HashSet(StringComparer.OrdinalIgnoreCase); + var remainingCapacity = _options.MaxAdvisoriesPerFetch; + + IReadOnlyList indexItems; + try + { + indexItems = await FetchIndexAsync(cancellationToken).ConfigureAwait(false); + } + catch (Exception ex) + { + _diagnostics.FetchFailure(); + _logger.LogError(ex, "Failed to retrieve VMware advisory index"); + await _stateRepository.MarkFailureAsync(SourceName, now, TimeSpan.FromMinutes(10), ex.Message, cancellationToken).ConfigureAwait(false); + throw; + } + + if (indexItems.Count == 0) + { + return; + } + + var orderedItems = indexItems + .Where(static item => !string.IsNullOrWhiteSpace(item.Id) && !string.IsNullOrWhiteSpace(item.DetailUrl)) + .OrderBy(static item => item.Modified ?? DateTimeOffset.MinValue) + .ThenBy(static item => item.Id, StringComparer.OrdinalIgnoreCase) + .ToArray(); + + var baseline = cursor.LastModified ?? now - _options.InitialBackfill; + var resumeStart = baseline - _options.ModifiedTolerance; + ProvenanceDiagnostics.ReportResumeWindow(SourceName, resumeStart, _logger); + var processedIds = new HashSet(cursor.ProcessedIds, StringComparer.OrdinalIgnoreCase); + var maxModified = cursor.LastModified ?? DateTimeOffset.MinValue; + var processedUpdated = false; + + foreach (var item in orderedItems) + { + if (remainingCapacity <= 0) + { + break; + } + + cancellationToken.ThrowIfCancellationRequested(); + + var modified = (item.Modified ?? DateTimeOffset.MinValue).ToUniversalTime(); + if (modified < baseline - _options.ModifiedTolerance) + { + continue; + } + + if (cursor.LastModified.HasValue && modified < cursor.LastModified.Value - _options.ModifiedTolerance) + { + continue; + } + + if (modified == cursor.LastModified && cursor.ProcessedIds.Contains(item.Id, StringComparer.OrdinalIgnoreCase)) + { + continue; + } + + if (!Uri.TryCreate(item.DetailUrl, UriKind.Absolute, out var detailUri)) + { + _logger.LogWarning("VMware advisory {AdvisoryId} has invalid detail URL {Url}", item.Id, item.DetailUrl); + continue; + } + + var cacheKey = detailUri.AbsoluteUri; + touchedResources.Add(cacheKey); + + var existing = await _documentStore.FindBySourceAndUriAsync(SourceName, cacheKey, cancellationToken).ConfigureAwait(false); + var metadata = new Dictionary(StringComparer.Ordinal) + { + ["vmware.id"] = item.Id, + ["vmware.modified"] = modified.ToString("O"), + }; + + SourceFetchResult result; + try + { + result = await _fetchService.FetchAsync( + new SourceFetchRequest(VmwareOptions.HttpClientName, SourceName, detailUri) + { + Metadata = metadata, + ETag = existing?.Etag, + LastModified = existing?.LastModified, + AcceptHeaders = new[] { "application/json" }, + }, + cancellationToken).ConfigureAwait(false); + } + catch (Exception ex) + { + _diagnostics.FetchFailure(); + _logger.LogError(ex, "Failed to fetch VMware advisory {AdvisoryId}", item.Id); + await _stateRepository.MarkFailureAsync(SourceName, now, TimeSpan.FromMinutes(5), ex.Message, cancellationToken).ConfigureAwait(false); + throw; + } + + if (result.IsNotModified) + { + _diagnostics.FetchUnchanged(); + if (existing is not null) + { + fetchCache[cacheKey] = VmwareFetchCacheEntry.FromDocument(existing); + pendingDocuments.Remove(existing.Id); + pendingMappings.Remove(existing.Id); + _logger.LogInformation("VMware advisory {AdvisoryId} returned 304 Not Modified", item.Id); + } + + continue; + } + + if (!result.IsSuccess || result.Document is null) + { + _diagnostics.FetchFailure(); + continue; + } + + remainingCapacity--; + + if (modified > maxModified) + { + maxModified = modified; + processedIds.Clear(); + processedUpdated = true; + } + + if (modified == maxModified) + { + processedIds.Add(item.Id); + processedUpdated = true; + } + + var cacheEntry = VmwareFetchCacheEntry.FromDocument(result.Document); + + if (existing is not null + && string.Equals(existing.Status, DocumentStatuses.Mapped, StringComparison.Ordinal) + && cursor.TryGetFetchCache(cacheKey, out var cachedEntry) + && cachedEntry.Matches(result.Document)) + { + _diagnostics.FetchUnchanged(); + fetchCache[cacheKey] = cacheEntry; + pendingDocuments.Remove(result.Document.Id); + pendingMappings.Remove(result.Document.Id); + await _documentStore.UpdateStatusAsync(result.Document.Id, existing.Status, cancellationToken).ConfigureAwait(false); + _logger.LogInformation("VMware advisory {AdvisoryId} unchanged; skipping reprocessing", item.Id); + continue; + } + + _diagnostics.FetchItem(); + fetchCache[cacheKey] = cacheEntry; + pendingDocuments.Add(result.Document.Id); + _logger.LogInformation( + "VMware advisory {AdvisoryId} fetched (documentId={DocumentId}, sha256={Sha})", + item.Id, + result.Document.Id, + result.Document.Sha256); + + if (_options.RequestDelay > TimeSpan.Zero) + { + try + { + await Task.Delay(_options.RequestDelay, cancellationToken).ConfigureAwait(false); + } + catch (TaskCanceledException) + { + break; + } + } + } + + if (fetchCache.Count > 0 && touchedResources.Count > 0) + { + var stale = fetchCache.Keys.Where(key => !touchedResources.Contains(key)).ToArray(); + foreach (var key in stale) + { + fetchCache.Remove(key); + } + } + + var updatedCursor = cursor + .WithPendingDocuments(pendingDocuments) + .WithPendingMappings(pendingMappings) + .WithFetchCache(fetchCache); + + if (processedUpdated) + { + updatedCursor = updatedCursor.WithLastModified(maxModified, processedIds); + } + + await UpdateCursorAsync(updatedCursor, cancellationToken).ConfigureAwait(false); + } + + public async Task ParseAsync(IServiceProvider services, CancellationToken cancellationToken) + { + ArgumentNullException.ThrowIfNull(services); + + var cursor = await GetCursorAsync(cancellationToken).ConfigureAwait(false); + if (cursor.PendingDocuments.Count == 0) + { + return; + } + + var remaining = cursor.PendingDocuments.ToList(); + var pendingMappings = cursor.PendingMappings.ToList(); + + foreach (var documentId in cursor.PendingDocuments) + { + cancellationToken.ThrowIfCancellationRequested(); + + var document = await _documentStore.FindAsync(documentId, cancellationToken).ConfigureAwait(false); + if (document is null) + { + remaining.Remove(documentId); + continue; + } + + if (!document.GridFsId.HasValue) + { + _logger.LogWarning("VMware document {DocumentId} missing GridFS payload", document.Id); + await _documentStore.UpdateStatusAsync(document.Id, DocumentStatuses.Failed, cancellationToken).ConfigureAwait(false); + remaining.Remove(documentId); + _diagnostics.ParseFailure(); + continue; + } + + byte[] bytes; + try + { + bytes = await _rawDocumentStorage.DownloadAsync(document.GridFsId.Value, cancellationToken).ConfigureAwait(false); + } + catch (Exception ex) + { + _logger.LogError(ex, "Failed downloading VMware document {DocumentId}", document.Id); + throw; + } + + VmwareDetailDto? detail; + try + { + detail = JsonSerializer.Deserialize(bytes, SerializerOptions); + } + catch (Exception ex) + { + _logger.LogWarning(ex, "Failed to deserialize VMware advisory {DocumentId}", document.Id); + await _documentStore.UpdateStatusAsync(document.Id, DocumentStatuses.Failed, cancellationToken).ConfigureAwait(false); + remaining.Remove(documentId); + _diagnostics.ParseFailure(); + continue; + } + + if (detail is null || string.IsNullOrWhiteSpace(detail.AdvisoryId)) + { + _logger.LogWarning("VMware advisory document {DocumentId} contained empty payload", document.Id); + await _documentStore.UpdateStatusAsync(document.Id, DocumentStatuses.Failed, cancellationToken).ConfigureAwait(false); + remaining.Remove(documentId); + _diagnostics.ParseFailure(); + continue; + } + + var sanitized = JsonSerializer.Serialize(detail, SerializerOptions); + var payload = MongoDB.Bson.BsonDocument.Parse(sanitized); + var dtoRecord = new DtoRecord(Guid.NewGuid(), document.Id, SourceName, "vmware.v1", payload, _timeProvider.GetUtcNow()); + + await _dtoStore.UpsertAsync(dtoRecord, cancellationToken).ConfigureAwait(false); + await _documentStore.UpdateStatusAsync(document.Id, DocumentStatuses.PendingMap, cancellationToken).ConfigureAwait(false); + + remaining.Remove(documentId); + if (!pendingMappings.Contains(documentId)) + { + pendingMappings.Add(documentId); + } + } + + var updatedCursor = cursor + .WithPendingDocuments(remaining) + .WithPendingMappings(pendingMappings); + + await UpdateCursorAsync(updatedCursor, cancellationToken).ConfigureAwait(false); + } + + public async Task MapAsync(IServiceProvider services, CancellationToken cancellationToken) + { + ArgumentNullException.ThrowIfNull(services); + + var cursor = await GetCursorAsync(cancellationToken).ConfigureAwait(false); + if (cursor.PendingMappings.Count == 0) + { + return; + } + + var pendingMappings = cursor.PendingMappings.ToList(); + + foreach (var documentId in cursor.PendingMappings) + { + cancellationToken.ThrowIfCancellationRequested(); + + var dto = await _dtoStore.FindByDocumentIdAsync(documentId, cancellationToken).ConfigureAwait(false); + var document = await _documentStore.FindAsync(documentId, cancellationToken).ConfigureAwait(false); + if (dto is null || document is null) + { + pendingMappings.Remove(documentId); + continue; + } + + var json = dto.Payload.ToJson(new JsonWriterSettings + { + OutputMode = JsonOutputMode.RelaxedExtendedJson, + }); + + VmwareDetailDto? detail; + try + { + detail = JsonSerializer.Deserialize(json, SerializerOptions); + } + catch (Exception ex) + { + _logger.LogError(ex, "Failed to deserialize VMware DTO for document {DocumentId}", document.Id); + await _documentStore.UpdateStatusAsync(document.Id, DocumentStatuses.Failed, cancellationToken).ConfigureAwait(false); + pendingMappings.Remove(documentId); + continue; + } + + if (detail is null || string.IsNullOrWhiteSpace(detail.AdvisoryId)) + { + await _documentStore.UpdateStatusAsync(document.Id, DocumentStatuses.Failed, cancellationToken).ConfigureAwait(false); + pendingMappings.Remove(documentId); + continue; + } + + var (advisory, flag) = VmwareMapper.Map(detail, document, dto); + await _advisoryStore.UpsertAsync(advisory, cancellationToken).ConfigureAwait(false); + await _psirtFlagStore.UpsertAsync(flag, cancellationToken).ConfigureAwait(false); + await _documentStore.UpdateStatusAsync(document.Id, DocumentStatuses.Mapped, cancellationToken).ConfigureAwait(false); + _diagnostics.MapAffectedCount(advisory.AffectedPackages.Length); + _logger.LogInformation( + "VMware advisory {AdvisoryId} mapped with {AffectedCount} affected packages", + detail.AdvisoryId, + advisory.AffectedPackages.Length); + + pendingMappings.Remove(documentId); + } + + var updatedCursor = cursor.WithPendingMappings(pendingMappings); + await UpdateCursorAsync(updatedCursor, cancellationToken).ConfigureAwait(false); + } + + private async Task> FetchIndexAsync(CancellationToken cancellationToken) + { + var client = _httpClientFactory.CreateClient(VmwareOptions.HttpClientName); + using var response = await client.GetAsync(_options.IndexUri, cancellationToken).ConfigureAwait(false); + response.EnsureSuccessStatusCode(); + + await using var stream = await response.Content.ReadAsStreamAsync(cancellationToken).ConfigureAwait(false); + var items = await JsonSerializer.DeserializeAsync>(stream, SerializerOptions, cancellationToken).ConfigureAwait(false); + return items ?? Array.Empty(); + } + + private async Task GetCursorAsync(CancellationToken cancellationToken) + { + var state = await _stateRepository.TryGetAsync(SourceName, cancellationToken).ConfigureAwait(false); + return state is null ? VmwareCursor.Empty : VmwareCursor.FromBson(state.Cursor); + } + + private async Task UpdateCursorAsync(VmwareCursor cursor, CancellationToken cancellationToken) + { + var document = cursor.ToBsonDocument(); + await _stateRepository.UpdateCursorAsync(SourceName, document, _timeProvider.GetUtcNow(), cancellationToken).ConfigureAwait(false); + } +} diff --git a/src/StellaOps.Feedser.Source.Vndr.Vmware/VmwareConnectorPlugin.cs b/src/StellaOps.Feedser.Source.Vndr.Vmware/VmwareConnectorPlugin.cs new file mode 100644 index 00000000..98f53d16 --- /dev/null +++ b/src/StellaOps.Feedser.Source.Vndr.Vmware/VmwareConnectorPlugin.cs @@ -0,0 +1,20 @@ +using System; +using Microsoft.Extensions.DependencyInjection; +using StellaOps.Plugin; + +namespace StellaOps.Feedser.Source.Vndr.Vmware; + +public sealed class VmwareConnectorPlugin : IConnectorPlugin +{ + public string Name => SourceName; + + public static string SourceName => "vmware"; + + public bool IsAvailable(IServiceProvider services) => services is not null; + + public IFeedConnector Create(IServiceProvider services) + { + ArgumentNullException.ThrowIfNull(services); + return ActivatorUtilities.CreateInstance(services); + } +} diff --git a/src/StellaOps.Feedser.Source.Vndr.Vmware/VmwareDependencyInjectionRoutine.cs b/src/StellaOps.Feedser.Source.Vndr.Vmware/VmwareDependencyInjectionRoutine.cs new file mode 100644 index 00000000..914aceb7 --- /dev/null +++ b/src/StellaOps.Feedser.Source.Vndr.Vmware/VmwareDependencyInjectionRoutine.cs @@ -0,0 +1,53 @@ +using System; +using Microsoft.Extensions.Configuration; +using Microsoft.Extensions.DependencyInjection; +using StellaOps.DependencyInjection; +using StellaOps.Feedser.Core.Jobs; +using StellaOps.Feedser.Source.Vndr.Vmware.Configuration; + +namespace StellaOps.Feedser.Source.Vndr.Vmware; + +public sealed class VmwareDependencyInjectionRoutine : IDependencyInjectionRoutine +{ + private const string ConfigurationSection = "feedser:sources:vmware"; + private const string FetchCron = "10,40 * * * *"; + private const string ParseCron = "15,45 * * * *"; + private const string MapCron = "20,50 * * * *"; + + private static readonly TimeSpan FetchTimeout = TimeSpan.FromMinutes(10); + private static readonly TimeSpan ParseTimeout = TimeSpan.FromMinutes(10); + private static readonly TimeSpan MapTimeout = TimeSpan.FromMinutes(15); + private static readonly TimeSpan LeaseDuration = TimeSpan.FromMinutes(5); + + public IServiceCollection Register(IServiceCollection services, IConfiguration configuration) + { + ArgumentNullException.ThrowIfNull(services); + ArgumentNullException.ThrowIfNull(configuration); + + services.AddVmwareConnector(options => + { + configuration.GetSection(ConfigurationSection).Bind(options); + options.Validate(); + }); + + var scheduler = new JobSchedulerBuilder(services); + scheduler + .AddJob( + VmwareJobKinds.Fetch, + cronExpression: FetchCron, + timeout: FetchTimeout, + leaseDuration: LeaseDuration) + .AddJob( + VmwareJobKinds.Parse, + cronExpression: ParseCron, + timeout: ParseTimeout, + leaseDuration: LeaseDuration) + .AddJob( + VmwareJobKinds.Map, + cronExpression: MapCron, + timeout: MapTimeout, + leaseDuration: LeaseDuration); + + return services; + } +} diff --git a/src/StellaOps.Feedser.Source.Vndr.Vmware/VmwareDiagnostics.cs b/src/StellaOps.Feedser.Source.Vndr.Vmware/VmwareDiagnostics.cs new file mode 100644 index 00000000..57a68e6b --- /dev/null +++ b/src/StellaOps.Feedser.Source.Vndr.Vmware/VmwareDiagnostics.cs @@ -0,0 +1,67 @@ +using System; +using System.Diagnostics.Metrics; + +namespace StellaOps.Feedser.Source.Vndr.Vmware; + +/// +/// VMware connector metrics (fetch, parse, map). +/// +public sealed class VmwareDiagnostics : IDisposable +{ + public const string MeterName = "StellaOps.Feedser.Source.Vndr.Vmware"; + private const string MeterVersion = "1.0.0"; + + private readonly Meter _meter; + private readonly Counter _fetchItems; + private readonly Counter _fetchFailures; + private readonly Counter _fetchUnchanged; + private readonly Counter _parseFailures; + private readonly Histogram _mapAffectedCount; + + public VmwareDiagnostics() + { + _meter = new Meter(MeterName, MeterVersion); + _fetchItems = _meter.CreateCounter( + name: "vmware.fetch.items", + unit: "documents", + description: "Number of VMware advisory documents fetched."); + _fetchFailures = _meter.CreateCounter( + name: "vmware.fetch.failures", + unit: "operations", + description: "Number of VMware fetch failures."); + _fetchUnchanged = _meter.CreateCounter( + name: "vmware.fetch.unchanged", + unit: "documents", + description: "Number of VMware advisories skipped due to unchanged content."); + _parseFailures = _meter.CreateCounter( + name: "vmware.parse.fail", + unit: "documents", + description: "Number of VMware advisory documents that failed to parse."); + _mapAffectedCount = _meter.CreateHistogram( + name: "vmware.map.affected_count", + unit: "packages", + description: "Distribution of affected-package counts emitted per VMware advisory."); + } + + public void FetchItem() => _fetchItems.Add(1); + + public void FetchFailure() => _fetchFailures.Add(1); + + public void FetchUnchanged() => _fetchUnchanged.Add(1); + + public void ParseFailure() => _parseFailures.Add(1); + + public void MapAffectedCount(int count) + { + if (count < 0) + { + return; + } + + _mapAffectedCount.Record(count); + } + + public Meter Meter => _meter; + + public void Dispose() => _meter.Dispose(); +} diff --git a/src/StellaOps.Feedser.Source.Vndr.Vmware/VmwareServiceCollectionExtensions.cs b/src/StellaOps.Feedser.Source.Vndr.Vmware/VmwareServiceCollectionExtensions.cs new file mode 100644 index 00000000..7765876e --- /dev/null +++ b/src/StellaOps.Feedser.Source.Vndr.Vmware/VmwareServiceCollectionExtensions.cs @@ -0,0 +1,39 @@ +using System; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.DependencyInjection.Extensions; +using Microsoft.Extensions.Options; +using StellaOps.Feedser.Source.Common.Http; +using StellaOps.Feedser.Source.Vndr.Vmware.Configuration; + +namespace StellaOps.Feedser.Source.Vndr.Vmware; + +public static class VmwareServiceCollectionExtensions +{ + public static IServiceCollection AddVmwareConnector(this IServiceCollection services, Action configure) + { + ArgumentNullException.ThrowIfNull(services); + ArgumentNullException.ThrowIfNull(configure); + + services.AddOptions() + .Configure(configure) + .PostConfigure(static opts => opts.Validate()); + + services.AddSourceHttpClient(VmwareOptions.HttpClientName, (sp, clientOptions) => + { + var options = sp.GetRequiredService>().Value; + clientOptions.BaseAddress = new Uri(options.IndexUri.GetLeftPart(UriPartial.Authority)); + clientOptions.Timeout = options.HttpTimeout; + clientOptions.UserAgent = "StellaOps.Feedser.VMware/1.0"; + clientOptions.AllowedHosts.Clear(); + clientOptions.AllowedHosts.Add(options.IndexUri.Host); + clientOptions.DefaultRequestHeaders["Accept"] = "application/json"; + }); + + services.TryAddSingleton(); + services.AddTransient(); + services.AddTransient(); + services.AddTransient(); + services.AddTransient(); + return services; + } +} diff --git a/src/StellaOps.Feedser.Storage.Mongo.Tests/AdvisoryStorePerformanceTests.cs b/src/StellaOps.Feedser.Storage.Mongo.Tests/AdvisoryStorePerformanceTests.cs new file mode 100644 index 00000000..f6a7d744 --- /dev/null +++ b/src/StellaOps.Feedser.Storage.Mongo.Tests/AdvisoryStorePerformanceTests.cs @@ -0,0 +1,195 @@ +using System.Diagnostics; +using System.Linq; +using System.Threading; +using Microsoft.Extensions.Logging.Abstractions; +using Microsoft.Extensions.Options; +using StellaOps.Feedser.Models; +using StellaOps.Feedser.Storage.Mongo; +using StellaOps.Feedser.Storage.Mongo.Advisories; +using StellaOps.Feedser.Storage.Mongo.Aliases; +using StellaOps.Feedser.Storage.Mongo.Migrations; +using Xunit; +using Xunit.Abstractions; + +namespace StellaOps.Feedser.Storage.Mongo.Tests; + +[Collection("mongo-fixture")] +public sealed class AdvisoryStorePerformanceTests : IClassFixture +{ + private const int LargeAdvisoryCount = 30; + private const int AliasesPerAdvisory = 24; + private const int ReferencesPerAdvisory = 180; + private const int AffectedPackagesPerAdvisory = 140; + private const int VersionRangesPerPackage = 4; + private const int CvssMetricsPerAdvisory = 24; + private const int ProvenanceEntriesPerAdvisory = 16; + private static readonly string LargeSummary = new('A', 128 * 1024); + private static readonly DateTimeOffset BasePublished = new(2024, 1, 1, 0, 0, 0, TimeSpan.Zero); + private static readonly DateTimeOffset BaseRecorded = new(2024, 1, 1, 0, 0, 0, TimeSpan.Zero); + private static readonly TimeSpan TotalBudget = TimeSpan.FromSeconds(28); + private const double UpsertBudgetPerAdvisoryMs = 500; + private const double FetchBudgetPerAdvisoryMs = 200; + private const double FindBudgetPerAdvisoryMs = 200; + + private readonly MongoIntegrationFixture _fixture; + private readonly ITestOutputHelper _output; + + public AdvisoryStorePerformanceTests(MongoIntegrationFixture fixture, ITestOutputHelper output) + { + _fixture = fixture; + _output = output; + } + + [Fact] + public async Task UpsertAndQueryLargeAdvisories_CompletesWithinBudget() + { + var databaseName = $"feedser-performance-{Guid.NewGuid():N}"; + var database = _fixture.Client.GetDatabase(databaseName); + + try + { + var migrationRunner = new MongoMigrationRunner( + database, + Array.Empty(), + NullLogger.Instance, + TimeProvider.System); + + var bootstrapper = new MongoBootstrapper( + database, + Options.Create(new MongoStorageOptions()), + NullLogger.Instance, + migrationRunner); + await bootstrapper.InitializeAsync(CancellationToken.None); + + var aliasStore = new AliasStore(database, NullLogger.Instance); + var store = new AdvisoryStore(database, aliasStore, NullLogger.Instance, TimeProvider.System); + using var cts = new CancellationTokenSource(TimeSpan.FromSeconds(45)); + + // Warm up collections (indexes, serialization caches) so perf timings exclude one-time setup work. + var warmup = CreateLargeAdvisory(-1); + await store.UpsertAsync(warmup, cts.Token); + _ = await store.FindAsync(warmup.AdvisoryKey, cts.Token); + _ = await store.GetRecentAsync(1, cts.Token); + + var advisories = Enumerable.Range(0, LargeAdvisoryCount) + .Select(CreateLargeAdvisory) + .ToArray(); + + var upsertWatch = Stopwatch.StartNew(); + foreach (var advisory in advisories) + { + await store.UpsertAsync(advisory, cts.Token); + } + + upsertWatch.Stop(); + var upsertPerAdvisory = upsertWatch.Elapsed.TotalMilliseconds / LargeAdvisoryCount; + + var fetchWatch = Stopwatch.StartNew(); + var recent = await store.GetRecentAsync(LargeAdvisoryCount, cts.Token); + fetchWatch.Stop(); + var fetchPerAdvisory = fetchWatch.Elapsed.TotalMilliseconds / LargeAdvisoryCount; + + Assert.Equal(LargeAdvisoryCount, recent.Count); + + var findWatch = Stopwatch.StartNew(); + foreach (var advisory in advisories) + { + var fetched = await store.FindAsync(advisory.AdvisoryKey, cts.Token); + Assert.NotNull(fetched); + } + + findWatch.Stop(); + var findPerAdvisory = findWatch.Elapsed.TotalMilliseconds / LargeAdvisoryCount; + + var totalElapsed = upsertWatch.Elapsed + fetchWatch.Elapsed + findWatch.Elapsed; + + _output.WriteLine($"Upserted {LargeAdvisoryCount} large advisories in {upsertWatch.Elapsed} ({upsertPerAdvisory:F2} ms/doc)."); + _output.WriteLine($"Fetched recent advisories in {fetchWatch.Elapsed} ({fetchPerAdvisory:F2} ms/doc)."); + _output.WriteLine($"Looked up advisories individually in {findWatch.Elapsed} ({findPerAdvisory:F2} ms/doc)."); + _output.WriteLine($"Total elapsed {totalElapsed}."); + + Assert.True(upsertPerAdvisory <= UpsertBudgetPerAdvisoryMs, $"Upsert exceeded {UpsertBudgetPerAdvisoryMs} ms per advisory: {upsertPerAdvisory:F2} ms."); + Assert.True(fetchPerAdvisory <= FetchBudgetPerAdvisoryMs, $"GetRecent exceeded {FetchBudgetPerAdvisoryMs} ms per advisory: {fetchPerAdvisory:F2} ms."); + Assert.True(findPerAdvisory <= FindBudgetPerAdvisoryMs, $"Find exceeded {FindBudgetPerAdvisoryMs} ms per advisory: {findPerAdvisory:F2} ms."); + Assert.True(totalElapsed <= TotalBudget, $"Mongo advisory operations exceeded total budget {TotalBudget}: {totalElapsed}."); + } + finally + { + await _fixture.Client.DropDatabaseAsync(databaseName); + } + } + + private static Advisory CreateLargeAdvisory(int index) + { + var baseKey = $"ADV-LARGE-{index:D4}"; + var published = BasePublished.AddDays(index); + var modified = published.AddHours(6); + + var aliases = Enumerable.Range(0, AliasesPerAdvisory) + .Select(i => $"ALIAS-{baseKey}-{i:D4}") + .ToArray(); + + var provenance = Enumerable.Range(0, ProvenanceEntriesPerAdvisory) + .Select(i => new AdvisoryProvenance( + source: i % 2 == 0 ? "nvd" : "vendor", + kind: i % 3 == 0 ? "normalized" : "enriched", + value: $"prov-{baseKey}-{i:D3}", + recordedAt: BaseRecorded.AddDays(i))) + .ToArray(); + + var references = Enumerable.Range(0, ReferencesPerAdvisory) + .Select(i => new AdvisoryReference( + url: $"https://vuln.example.com/{baseKey}/ref/{i:D4}", + kind: i % 2 == 0 ? "advisory" : "article", + sourceTag: $"tag-{i % 7}", + summary: $"Reference {baseKey} #{i}", + provenance: provenance[i % provenance.Length])) + .ToArray(); + + var affectedPackages = Enumerable.Range(0, AffectedPackagesPerAdvisory) + .Select(i => new AffectedPackage( + type: i % 3 == 0 ? AffectedPackageTypes.Rpm : AffectedPackageTypes.Deb, + identifier: $"pkg/{baseKey}/{i:D4}", + platform: i % 4 == 0 ? "linux/x86_64" : "linux/aarch64", + versionRanges: Enumerable.Range(0, VersionRangesPerPackage) + .Select(r => new AffectedVersionRange( + rangeKind: r % 2 == 0 ? "semver" : "evr", + introducedVersion: $"1.{index}.{i}.{r}", + fixedVersion: $"2.{index}.{i}.{r}", + lastAffectedVersion: $"1.{index}.{i}.{r}", + rangeExpression: $">=1.{index}.{i}.{r} <2.{index}.{i}.{r}", + provenance: provenance[(i + r) % provenance.Length])) + .ToArray(), + statuses: Array.Empty(), + provenance: new[] + { + provenance[i % provenance.Length], + provenance[(i + 3) % provenance.Length], + })) + .ToArray(); + + var cvssMetrics = Enumerable.Range(0, CvssMetricsPerAdvisory) + .Select(i => new CvssMetric( + version: i % 2 == 0 ? "3.1" : "2.0", + vector: $"CVSS:3.1/AV:N/AC:L/PR:N/UI:N/S:U/C:H/I:H/A:{(i % 3 == 0 ? "H" : "L")}", + baseScore: Math.Max(0, 9.8 - i * 0.2), + baseSeverity: i % 3 == 0 ? "critical" : "high", + provenance: provenance[i % provenance.Length])) + .ToArray(); + + return new Advisory( + advisoryKey: baseKey, + title: $"Large advisory {baseKey}", + summary: LargeSummary, + language: "en", + published: published, + modified: modified, + severity: "critical", + exploitKnown: index % 2 == 0, + aliases: aliases, + references: references, + affectedPackages: affectedPackages, + cvssMetrics: cvssMetrics, + provenance: provenance); + } +} diff --git a/src/StellaOps.Feedser.Storage.Mongo.Tests/AdvisoryStoreTests.cs b/src/StellaOps.Feedser.Storage.Mongo.Tests/AdvisoryStoreTests.cs new file mode 100644 index 00000000..68a45fa6 --- /dev/null +++ b/src/StellaOps.Feedser.Storage.Mongo.Tests/AdvisoryStoreTests.cs @@ -0,0 +1,162 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using Microsoft.Extensions.Logging.Abstractions; +using MongoDB.Driver; +using StellaOps.Feedser.Models; +using StellaOps.Feedser.Storage.Mongo.Advisories; +using StellaOps.Feedser.Storage.Mongo.Aliases; + +namespace StellaOps.Feedser.Storage.Mongo.Tests; + +[Collection("mongo-fixture")] +public sealed class AdvisoryStoreTests : IClassFixture +{ + private readonly MongoIntegrationFixture _fixture; + + public AdvisoryStoreTests(MongoIntegrationFixture fixture) + { + _fixture = fixture; + } + + [Fact] + public async Task UpsertAndFetchAdvisory() + { + await DropCollectionAsync(MongoStorageDefaults.Collections.Advisory); + await DropCollectionAsync(MongoStorageDefaults.Collections.Alias); + + var aliasStore = new AliasStore(_fixture.Database, NullLogger.Instance); + var store = new AdvisoryStore(_fixture.Database, aliasStore, NullLogger.Instance, TimeProvider.System); + var advisory = new Advisory( + advisoryKey: "ADV-1", + title: "Sample Advisory", + summary: "Demo", + language: "en", + published: DateTimeOffset.UtcNow, + modified: DateTimeOffset.UtcNow, + severity: "medium", + exploitKnown: false, + aliases: new[] { "ALIAS-1" }, + references: Array.Empty(), + affectedPackages: Array.Empty(), + cvssMetrics: Array.Empty(), + provenance: Array.Empty()); + + await store.UpsertAsync(advisory, CancellationToken.None); + + var fetched = await store.FindAsync("ADV-1", CancellationToken.None); + Assert.NotNull(fetched); + Assert.Equal(advisory.AdvisoryKey, fetched!.AdvisoryKey); + + var recent = await store.GetRecentAsync(5, CancellationToken.None); + Assert.NotEmpty(recent); + + var aliases = await aliasStore.GetByAdvisoryAsync("ADV-1", CancellationToken.None); + Assert.Contains(aliases, record => record.Scheme == AliasStoreConstants.PrimaryScheme && record.Value == "ADV-1"); + Assert.Contains(aliases, record => record.Value == "ALIAS-1"); + } + + [Fact] + public async Task RangePrimitives_RoundTripThroughMongo() + { + await DropCollectionAsync(MongoStorageDefaults.Collections.Advisory); + await DropCollectionAsync(MongoStorageDefaults.Collections.Alias); + + var aliasStore = new AliasStore(_fixture.Database, NullLogger.Instance); + var store = new AdvisoryStore(_fixture.Database, aliasStore, NullLogger.Instance, TimeProvider.System); + + var recordedAt = new DateTimeOffset(2025, 1, 1, 0, 0, 0, TimeSpan.Zero); + var provenance = new AdvisoryProvenance("source-x", "mapper", "payload-123", recordedAt); + var rangePrimitives = new RangePrimitives( + new SemVerPrimitive( + Introduced: "1.0.0", + IntroducedInclusive: true, + Fixed: "1.2.0", + FixedInclusive: false, + LastAffected: "1.1.5", + LastAffectedInclusive: true, + ConstraintExpression: ">=1.0.0 <1.2.0"), + new NevraPrimitive( + Introduced: new NevraComponent("pkg", 0, "1.0.0", "1", "x86_64"), + Fixed: new NevraComponent("pkg", 1, "1.2.0", "2", "x86_64"), + LastAffected: null), + new EvrPrimitive( + Introduced: new EvrComponent(1, "1.0.0", "1"), + Fixed: null, + LastAffected: new EvrComponent(1, "1.1.5", null)), + new Dictionary(StringComparer.Ordinal) + { + ["channel"] = "stable", + ["notesHash"] = "abc123", + }); + + var versionRange = new AffectedVersionRange( + rangeKind: "semver", + introducedVersion: "1.0.0", + fixedVersion: "1.2.0", + lastAffectedVersion: "1.1.5", + rangeExpression: ">=1.0.0 <1.2.0", + provenance, + rangePrimitives); + + var affectedPackage = new AffectedPackage( + type: "semver", + identifier: "pkg@1.x", + platform: "linux", + versionRanges: new[] { versionRange }, + statuses: Array.Empty(), + provenance: new[] { provenance }); + + var advisory = new Advisory( + advisoryKey: "ADV-RANGE-1", + title: "Sample Range Primitive", + summary: "Testing range primitive persistence.", + language: "en", + published: recordedAt, + modified: recordedAt, + severity: "medium", + exploitKnown: false, + aliases: new[] { "CVE-2025-0001" }, + references: Array.Empty(), + affectedPackages: new[] { affectedPackage }, + cvssMetrics: Array.Empty(), + provenance: new[] { provenance }); + + await store.UpsertAsync(advisory, CancellationToken.None); + + var fetched = await store.FindAsync("ADV-RANGE-1", CancellationToken.None); + Assert.NotNull(fetched); + var fetchedPackage = Assert.Single(fetched!.AffectedPackages); + var fetchedRange = Assert.Single(fetchedPackage.VersionRanges); + + Assert.Equal(versionRange.RangeKind, fetchedRange.RangeKind); + Assert.Equal(versionRange.IntroducedVersion, fetchedRange.IntroducedVersion); + Assert.Equal(versionRange.FixedVersion, fetchedRange.FixedVersion); + Assert.Equal(versionRange.LastAffectedVersion, fetchedRange.LastAffectedVersion); + Assert.Equal(versionRange.RangeExpression, fetchedRange.RangeExpression); + Assert.Equal(versionRange.Provenance.Source, fetchedRange.Provenance.Source); + Assert.Equal(versionRange.Provenance.Kind, fetchedRange.Provenance.Kind); + Assert.Equal(versionRange.Provenance.Value, fetchedRange.Provenance.Value); + Assert.Equal(versionRange.Provenance.DecisionReason, fetchedRange.Provenance.DecisionReason); + Assert.Equal(versionRange.Provenance.RecordedAt, fetchedRange.Provenance.RecordedAt); + Assert.True(versionRange.Provenance.FieldMask.SequenceEqual(fetchedRange.Provenance.FieldMask)); + + Assert.NotNull(fetchedRange.Primitives); + Assert.Equal(rangePrimitives.SemVer, fetchedRange.Primitives!.SemVer); + Assert.Equal(rangePrimitives.Nevra, fetchedRange.Primitives.Nevra); + Assert.Equal(rangePrimitives.Evr, fetchedRange.Primitives.Evr); + Assert.Equal(rangePrimitives.VendorExtensions, fetchedRange.Primitives.VendorExtensions); + } + + private async Task DropCollectionAsync(string collectionName) + { + try + { + await _fixture.Database.DropCollectionAsync(collectionName); + } + catch (MongoDB.Driver.MongoCommandException ex) when (ex.CodeName == "NamespaceNotFound" || ex.Message.Contains("ns not found", StringComparison.OrdinalIgnoreCase)) + { + // ignore missing collection + } + } +} diff --git a/src/StellaOps.Feedser.Storage.Mongo.Tests/AliasStoreTests.cs b/src/StellaOps.Feedser.Storage.Mongo.Tests/AliasStoreTests.cs new file mode 100644 index 00000000..122b29b3 --- /dev/null +++ b/src/StellaOps.Feedser.Storage.Mongo.Tests/AliasStoreTests.cs @@ -0,0 +1,60 @@ +using System; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.Extensions.Logging.Abstractions; +using MongoDB.Driver; +using StellaOps.Feedser.Storage.Mongo; +using StellaOps.Feedser.Storage.Mongo.Aliases; + +namespace StellaOps.Feedser.Storage.Mongo.Tests; + +[Collection("mongo-fixture")] +public sealed class AliasStoreTests : IClassFixture +{ + private readonly MongoIntegrationFixture _fixture; + + public AliasStoreTests(MongoIntegrationFixture fixture) + { + _fixture = fixture; + } + + [Fact] + public async Task ReplaceAsync_UpsertsAliases_AndDetectsCollision() + { + await DropAliasCollectionAsync(); + var store = new AliasStore(_fixture.Database, NullLogger.Instance); + + var timestamp = DateTimeOffset.UtcNow; + await store.ReplaceAsync( + "ADV-1", + new[] { new AliasEntry("CVE", "CVE-2025-1234"), new AliasEntry(AliasStoreConstants.PrimaryScheme, "ADV-1") }, + timestamp, + CancellationToken.None); + + var firstAliases = await store.GetByAdvisoryAsync("ADV-1", CancellationToken.None); + Assert.Contains(firstAliases, record => record.Scheme == "CVE" && record.Value == "CVE-2025-1234"); + + var result = await store.ReplaceAsync( + "ADV-2", + new[] { new AliasEntry("CVE", "CVE-2025-1234"), new AliasEntry(AliasStoreConstants.PrimaryScheme, "ADV-2") }, + timestamp.AddMinutes(1), + CancellationToken.None); + + Assert.NotEmpty(result.Collisions); + var collision = Assert.Single(result.Collisions); + Assert.Equal("CVE", collision.Scheme); + Assert.Contains("ADV-1", collision.AdvisoryKeys); + Assert.Contains("ADV-2", collision.AdvisoryKeys); + } + + private async Task DropAliasCollectionAsync() + { + try + { + await _fixture.Database.DropCollectionAsync(MongoStorageDefaults.Collections.Alias); + } + catch (MongoDB.Driver.MongoCommandException ex) when (ex.CodeName == "NamespaceNotFound" || ex.Message.Contains("ns not found", StringComparison.OrdinalIgnoreCase)) + { + } + } +} diff --git a/src/StellaOps.Feedser.Storage.Mongo.Tests/DocumentStoreTests.cs b/src/StellaOps.Feedser.Storage.Mongo.Tests/DocumentStoreTests.cs new file mode 100644 index 00000000..f4f9b2b5 --- /dev/null +++ b/src/StellaOps.Feedser.Storage.Mongo.Tests/DocumentStoreTests.cs @@ -0,0 +1,51 @@ +using Microsoft.Extensions.Logging.Abstractions; +using StellaOps.Feedser.Storage.Mongo.Documents; + +namespace StellaOps.Feedser.Storage.Mongo.Tests; + +[Collection("mongo-fixture")] +public sealed class DocumentStoreTests : IClassFixture +{ + private readonly MongoIntegrationFixture _fixture; + + public DocumentStoreTests(MongoIntegrationFixture fixture) + { + _fixture = fixture; + } + + [Fact] + public async Task UpsertAndLookupDocument() + { + var store = new DocumentStore(_fixture.Database, NullLogger.Instance); + var id = Guid.NewGuid(); + var record = new DocumentRecord( + id, + "source", + "https://example.com/advisory.json", + DateTimeOffset.UtcNow, + "sha123", + "pending", + "application/json", + new Dictionary { ["etag"] = "abc" }, + new Dictionary { ["note"] = "test" }, + "etag-value", + DateTimeOffset.UtcNow, + null, + DateTimeOffset.UtcNow.AddDays(30)); + + var upserted = await store.UpsertAsync(record, CancellationToken.None); + Assert.Equal(id, upserted.Id); + + var fetched = await store.FindBySourceAndUriAsync("source", "https://example.com/advisory.json", CancellationToken.None); + Assert.NotNull(fetched); + Assert.Equal("pending", fetched!.Status); + Assert.Equal("test", fetched.Metadata!["note"]); + + var statusUpdated = await store.UpdateStatusAsync(id, "processed", CancellationToken.None); + Assert.True(statusUpdated); + + var refreshed = await store.FindAsync(id, CancellationToken.None); + Assert.NotNull(refreshed); + Assert.Equal("processed", refreshed!.Status); + } +} diff --git a/src/StellaOps.Feedser.Storage.Mongo.Tests/DtoStoreTests.cs b/src/StellaOps.Feedser.Storage.Mongo.Tests/DtoStoreTests.cs new file mode 100644 index 00000000..4bdf309e --- /dev/null +++ b/src/StellaOps.Feedser.Storage.Mongo.Tests/DtoStoreTests.cs @@ -0,0 +1,40 @@ +using Microsoft.Extensions.Logging.Abstractions; +using MongoDB.Bson; +using StellaOps.Feedser.Storage.Mongo.Dtos; + +namespace StellaOps.Feedser.Storage.Mongo.Tests; + +[Collection("mongo-fixture")] +public sealed class DtoStoreTests : IClassFixture +{ + private readonly MongoIntegrationFixture _fixture; + + public DtoStoreTests(MongoIntegrationFixture fixture) + { + _fixture = fixture; + } + + [Fact] + public async Task UpsertAndLookupDto() + { + var store = new DtoStore(_fixture.Database, NullLogger.Instance); + var record = new DtoRecord( + Guid.NewGuid(), + Guid.NewGuid(), + "source", + "1.0", + new BsonDocument("value", 1), + DateTimeOffset.UtcNow); + + var upserted = await store.UpsertAsync(record, CancellationToken.None); + Assert.Equal(record.DocumentId, upserted.DocumentId); + + var fetched = await store.FindByDocumentIdAsync(record.DocumentId, CancellationToken.None); + Assert.NotNull(fetched); + Assert.Equal(1, fetched!.Payload["value"].AsInt32); + + var bySource = await store.GetBySourceAsync("source", 10, CancellationToken.None); + Assert.Single(bySource); + Assert.Equal(record.DocumentId, bySource[0].DocumentId); + } +} diff --git a/src/StellaOps.Feedser.Storage.Mongo.Tests/ExportStateManagerTests.cs b/src/StellaOps.Feedser.Storage.Mongo.Tests/ExportStateManagerTests.cs new file mode 100644 index 00000000..7e4dc9dd --- /dev/null +++ b/src/StellaOps.Feedser.Storage.Mongo.Tests/ExportStateManagerTests.cs @@ -0,0 +1,208 @@ +using System; +using System.Collections.Generic; +using System.Threading; +using System.Threading.Tasks; +using StellaOps.Feedser.Storage.Mongo.Exporting; + +namespace StellaOps.Feedser.Storage.Mongo.Tests; + +public sealed class ExportStateManagerTests +{ + [Fact] + public async Task StoreFullExportInitializesBaseline() + { + var store = new InMemoryExportStateStore(); + var timeProvider = new TestTimeProvider(DateTimeOffset.Parse("2024-07-20T12:00:00Z")); + var manager = new ExportStateManager(store, timeProvider); + + var record = await manager.StoreFullExportAsync( + exporterId: "export:json", + exportId: "20240720T120000Z", + exportDigest: "sha256:abcd", + cursor: "cursor-1", + targetRepository: "registry.local/json", + exporterVersion: "1.0.0", + resetBaseline: true, + manifest: Array.Empty(), + cancellationToken: CancellationToken.None); + + Assert.Equal("export:json", record.Id); + Assert.Equal("20240720T120000Z", record.BaseExportId); + Assert.Equal("sha256:abcd", record.BaseDigest); + Assert.Equal("sha256:abcd", record.LastFullDigest); + Assert.Null(record.LastDeltaDigest); + Assert.Equal("cursor-1", record.ExportCursor); + Assert.Equal("registry.local/json", record.TargetRepository); + Assert.Equal("1.0.0", record.ExporterVersion); + Assert.Equal(timeProvider.Now, record.UpdatedAt); + } + + [Fact] + public async Task StoreFullExport_ResetBaselineOverridesExisting() + { + var store = new InMemoryExportStateStore(); + var timeProvider = new TestTimeProvider(DateTimeOffset.Parse("2024-07-20T12:00:00Z")); + var manager = new ExportStateManager(store, timeProvider); + + await manager.StoreFullExportAsync( + exporterId: "export:json", + exportId: "20240720T120000Z", + exportDigest: "sha256:base", + cursor: "cursor-base", + targetRepository: null, + exporterVersion: "1.0.0", + resetBaseline: true, + manifest: Array.Empty(), + cancellationToken: CancellationToken.None); + + timeProvider.Advance(TimeSpan.FromMinutes(5)); + var withoutReset = await manager.StoreFullExportAsync( + exporterId: "export:json", + exportId: "20240720T120500Z", + exportDigest: "sha256:new", + cursor: "cursor-new", + targetRepository: null, + exporterVersion: "1.0.1", + resetBaseline: false, + manifest: Array.Empty(), + cancellationToken: CancellationToken.None); + + Assert.Equal("20240720T120000Z", withoutReset.BaseExportId); + Assert.Equal("sha256:base", withoutReset.BaseDigest); + Assert.Equal("sha256:new", withoutReset.LastFullDigest); + Assert.Equal("cursor-new", withoutReset.ExportCursor); + Assert.Equal(timeProvider.Now, withoutReset.UpdatedAt); + + timeProvider.Advance(TimeSpan.FromMinutes(5)); + var reset = await manager.StoreFullExportAsync( + exporterId: "export:json", + exportId: "20240720T121000Z", + exportDigest: "sha256:final", + cursor: "cursor-final", + targetRepository: null, + exporterVersion: "1.0.2", + resetBaseline: true, + manifest: Array.Empty(), + cancellationToken: CancellationToken.None); + + Assert.Equal("20240720T121000Z", reset.BaseExportId); + Assert.Equal("sha256:final", reset.BaseDigest); + Assert.Equal("sha256:final", reset.LastFullDigest); + Assert.Null(reset.LastDeltaDigest); + Assert.Equal("cursor-final", reset.ExportCursor); + Assert.Equal(timeProvider.Now, reset.UpdatedAt); + } + + [Fact] + public async Task StoreFullExport_ResetsBaselineWhenRepositoryChanges() + { + var store = new InMemoryExportStateStore(); + var timeProvider = new TestTimeProvider(DateTimeOffset.Parse("2024-07-21T08:00:00Z")); + var manager = new ExportStateManager(store, timeProvider); + + await manager.StoreFullExportAsync( + exporterId: "export:json", + exportId: "20240721T080000Z", + exportDigest: "sha256:base", + cursor: "cursor-base", + targetRepository: "registry/v1/json", + exporterVersion: "1.0.0", + resetBaseline: true, + manifest: Array.Empty(), + cancellationToken: CancellationToken.None); + + timeProvider.Advance(TimeSpan.FromMinutes(10)); + var updated = await manager.StoreFullExportAsync( + exporterId: "export:json", + exportId: "20240721T081000Z", + exportDigest: "sha256:new", + cursor: "cursor-new", + targetRepository: "registry/v2/json", + exporterVersion: "1.1.0", + resetBaseline: false, + manifest: Array.Empty(), + cancellationToken: CancellationToken.None); + + Assert.Equal("20240721T081000Z", updated.BaseExportId); + Assert.Equal("sha256:new", updated.BaseDigest); + Assert.Equal("sha256:new", updated.LastFullDigest); + Assert.Equal("registry/v2/json", updated.TargetRepository); + } + + [Fact] + public async Task StoreDeltaExportRequiresBaseline() + { + var store = new InMemoryExportStateStore(); + var manager = new ExportStateManager(store); + + await Assert.ThrowsAsync(() => manager.StoreDeltaExportAsync( + exporterId: "export:json", + deltaDigest: "sha256:def", + cursor: null, + exporterVersion: "1.0.1", + manifest: Array.Empty(), + cancellationToken: CancellationToken.None)); + } + + [Fact] + public async Task StoreDeltaExportUpdatesExistingState() + { + var store = new InMemoryExportStateStore(); + var timeProvider = new TestTimeProvider(DateTimeOffset.Parse("2024-07-20T12:00:00Z")); + var manager = new ExportStateManager(store, timeProvider); + + await manager.StoreFullExportAsync( + exporterId: "export:json", + exportId: "20240720T120000Z", + exportDigest: "sha256:abcd", + cursor: "cursor-1", + targetRepository: null, + exporterVersion: "1.0.0", + resetBaseline: true, + manifest: Array.Empty(), + cancellationToken: CancellationToken.None); + + timeProvider.Advance(TimeSpan.FromMinutes(10)); + var delta = await manager.StoreDeltaExportAsync( + exporterId: "export:json", + deltaDigest: "sha256:ef01", + cursor: "cursor-2", + exporterVersion: "1.0.1", + manifest: Array.Empty(), + cancellationToken: CancellationToken.None); + + Assert.Equal("sha256:ef01", delta.LastDeltaDigest); + Assert.Equal("cursor-2", delta.ExportCursor); + Assert.Equal("1.0.1", delta.ExporterVersion); + Assert.Equal(timeProvider.Now, delta.UpdatedAt); + Assert.Equal("sha256:abcd", delta.LastFullDigest); + } + + private sealed class InMemoryExportStateStore : IExportStateStore + { + private readonly Dictionary _records = new(StringComparer.Ordinal); + + public Task FindAsync(string id, CancellationToken cancellationToken) + { + _records.TryGetValue(id, out var record); + return Task.FromResult(record); + } + + public Task UpsertAsync(ExportStateRecord record, CancellationToken cancellationToken) + { + _records[record.Id] = record; + return Task.FromResult(record); + } + } + + private sealed class TestTimeProvider : TimeProvider + { + public TestTimeProvider(DateTimeOffset start) => Now = start; + + public DateTimeOffset Now { get; private set; } + + public void Advance(TimeSpan delta) => Now = Now.Add(delta); + + public override DateTimeOffset GetUtcNow() => Now; + } +} diff --git a/src/StellaOps.Feedser.Storage.Mongo.Tests/ExportStateStoreTests.cs b/src/StellaOps.Feedser.Storage.Mongo.Tests/ExportStateStoreTests.cs new file mode 100644 index 00000000..8eed8ef3 --- /dev/null +++ b/src/StellaOps.Feedser.Storage.Mongo.Tests/ExportStateStoreTests.cs @@ -0,0 +1,42 @@ +using System; +using Microsoft.Extensions.Logging.Abstractions; +using StellaOps.Feedser.Storage.Mongo.Exporting; + +namespace StellaOps.Feedser.Storage.Mongo.Tests; + +[Collection("mongo-fixture")] +public sealed class ExportStateStoreTests : IClassFixture +{ + private readonly MongoIntegrationFixture _fixture; + + public ExportStateStoreTests(MongoIntegrationFixture fixture) + { + _fixture = fixture; + } + + [Fact] + public async Task UpsertAndFetchExportState() + { + var store = new ExportStateStore(_fixture.Database, NullLogger.Instance); + var record = new ExportStateRecord( + Id: "json", + BaseExportId: "base", + BaseDigest: "sha-base", + LastFullDigest: "sha-full", + LastDeltaDigest: null, + ExportCursor: "cursor", + TargetRepository: "repo", + ExporterVersion: "1.0", + UpdatedAt: DateTimeOffset.UtcNow, + Files: Array.Empty()); + + var saved = await store.UpsertAsync(record, CancellationToken.None); + Assert.Equal("json", saved.Id); + Assert.Empty(saved.Files); + + var fetched = await store.FindAsync("json", CancellationToken.None); + Assert.NotNull(fetched); + Assert.Equal("sha-full", fetched!.LastFullDigest); + Assert.Empty(fetched.Files); + } +} diff --git a/src/StellaOps.Feedser.Storage.Mongo.Tests/MergeEventStoreTests.cs b/src/StellaOps.Feedser.Storage.Mongo.Tests/MergeEventStoreTests.cs new file mode 100644 index 00000000..3758cf73 --- /dev/null +++ b/src/StellaOps.Feedser.Storage.Mongo.Tests/MergeEventStoreTests.cs @@ -0,0 +1,35 @@ +using Microsoft.Extensions.Logging.Abstractions; +using StellaOps.Feedser.Storage.Mongo.MergeEvents; + +namespace StellaOps.Feedser.Storage.Mongo.Tests; + +[Collection("mongo-fixture")] +public sealed class MergeEventStoreTests : IClassFixture +{ + private readonly MongoIntegrationFixture _fixture; + + public MergeEventStoreTests(MongoIntegrationFixture fixture) + { + _fixture = fixture; + } + + [Fact] + public async Task AppendAndReadMergeEvents() + { + var store = new MergeEventStore(_fixture.Database, NullLogger.Instance); + var record = new MergeEventRecord( + Guid.NewGuid(), + "ADV-1", + new byte[] { 0x01 }, + new byte[] { 0x02 }, + DateTimeOffset.UtcNow, + new List { Guid.NewGuid() }, + Array.Empty()); + + await store.AppendAsync(record, CancellationToken.None); + + var recent = await store.GetRecentAsync("ADV-1", 10, CancellationToken.None); + Assert.Single(recent); + Assert.Equal(record.AfterHash, recent[0].AfterHash); + } +} diff --git a/src/StellaOps.Feedser.Storage.Mongo.Tests/Migrations/MongoMigrationRunnerTests.cs b/src/StellaOps.Feedser.Storage.Mongo.Tests/Migrations/MongoMigrationRunnerTests.cs new file mode 100644 index 00000000..9db94f41 --- /dev/null +++ b/src/StellaOps.Feedser.Storage.Mongo.Tests/Migrations/MongoMigrationRunnerTests.cs @@ -0,0 +1,238 @@ +using System; +using System.Linq; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.Extensions.Logging.Abstractions; +using Microsoft.Extensions.Options; +using MongoDB.Bson; +using MongoDB.Driver; +using StellaOps.Feedser.Storage.Mongo; +using StellaOps.Feedser.Storage.Mongo.Migrations; +using Xunit; + +namespace StellaOps.Feedser.Storage.Mongo.Tests.Migrations; + +[Collection("mongo-fixture")] +public sealed class MongoMigrationRunnerTests +{ + private readonly MongoIntegrationFixture _fixture; + + public MongoMigrationRunnerTests(MongoIntegrationFixture fixture) + { + _fixture = fixture; + } + + [Fact] + public async Task RunAsync_AppliesPendingMigrationsOnce() + { + var databaseName = $"feedser-migrations-{Guid.NewGuid():N}"; + var database = _fixture.Client.GetDatabase(databaseName); + await database.CreateCollectionAsync(MongoStorageDefaults.Collections.Migrations); + + try + { + var migration = new TestMigration(); + var runner = new MongoMigrationRunner( + database, + new IMongoMigration[] { migration }, + NullLogger.Instance, + TimeProvider.System); + + await runner.RunAsync(CancellationToken.None); + await runner.RunAsync(CancellationToken.None); + + Assert.Equal(1, migration.ApplyCount); + + var count = await database + .GetCollection(MongoStorageDefaults.Collections.Migrations) + .CountDocumentsAsync(FilterDefinition.Empty); + Assert.Equal(1, count); + } + finally + { + await _fixture.Client.DropDatabaseAsync(databaseName); + } + } + + [Fact] + public async Task EnsureDocumentExpiryIndexesMigration_CreatesTtlIndexWhenRetentionEnabled() + { + var databaseName = $"feedser-doc-ttl-{Guid.NewGuid():N}"; + var database = _fixture.Client.GetDatabase(databaseName); + await database.CreateCollectionAsync(MongoStorageDefaults.Collections.Document); + await database.CreateCollectionAsync(MongoStorageDefaults.Collections.Migrations); + + try + { + var options = Options.Create(new MongoStorageOptions + { + RawDocumentRetention = TimeSpan.FromDays(45), + RawDocumentRetentionTtlGrace = TimeSpan.FromHours(12), + }); + + var migration = new EnsureDocumentExpiryIndexesMigration(options); + var runner = new MongoMigrationRunner( + database, + new IMongoMigration[] { migration }, + NullLogger.Instance, + TimeProvider.System); + + await runner.RunAsync(CancellationToken.None); + + var indexes = await database + .GetCollection(MongoStorageDefaults.Collections.Document) + .Indexes.ListAsync(); + var indexList = await indexes.ToListAsync(); + + var ttlIndex = indexList.Single(x => x["name"].AsString == "document_expiresAt_ttl"); + Assert.Equal(0, ttlIndex["expireAfterSeconds"].ToDouble()); + Assert.True(ttlIndex["partialFilterExpression"].AsBsonDocument["expiresAt"].AsBsonDocument["$exists"].ToBoolean()); + } + finally + { + await _fixture.Client.DropDatabaseAsync(databaseName); + } + } + + [Fact] + public async Task EnsureDocumentExpiryIndexesMigration_DropsTtlIndexWhenRetentionDisabled() + { + var databaseName = $"feedser-doc-notl-{Guid.NewGuid():N}"; + var database = _fixture.Client.GetDatabase(databaseName); + await database.CreateCollectionAsync(MongoStorageDefaults.Collections.Document); + await database.CreateCollectionAsync(MongoStorageDefaults.Collections.Migrations); + + try + { + var collection = database.GetCollection(MongoStorageDefaults.Collections.Document); + var keys = Builders.IndexKeys.Ascending("expiresAt"); + var options = new CreateIndexOptions + { + Name = "document_expiresAt_ttl", + ExpireAfter = TimeSpan.Zero, + PartialFilterExpression = Builders.Filter.Exists("expiresAt", true), + }; + + await collection.Indexes.CreateOneAsync(new CreateIndexModel(keys, options)); + + var migration = new EnsureDocumentExpiryIndexesMigration(Options.Create(new MongoStorageOptions + { + RawDocumentRetention = TimeSpan.Zero, + })); + + var runner = new MongoMigrationRunner( + database, + new IMongoMigration[] { migration }, + NullLogger.Instance, + TimeProvider.System); + + await runner.RunAsync(CancellationToken.None); + + var indexes = await collection.Indexes.ListAsync(); + var indexList = await indexes.ToListAsync(); + + Assert.DoesNotContain(indexList, x => x["name"].AsString == "document_expiresAt_ttl"); + var nonTtl = indexList.Single(x => x["name"].AsString == "document_expiresAt"); + Assert.False(nonTtl.Contains("expireAfterSeconds")); + } + finally + { + await _fixture.Client.DropDatabaseAsync(databaseName); + } + } + + [Fact] + public async Task EnsureGridFsExpiryIndexesMigration_CreatesTtlIndexWhenRetentionEnabled() + { + var databaseName = $"feedser-gridfs-ttl-{Guid.NewGuid():N}"; + var database = _fixture.Client.GetDatabase(databaseName); + await database.CreateCollectionAsync("documents.files"); + await database.CreateCollectionAsync(MongoStorageDefaults.Collections.Migrations); + + try + { + var migration = new EnsureGridFsExpiryIndexesMigration(Options.Create(new MongoStorageOptions + { + RawDocumentRetention = TimeSpan.FromDays(30), + })); + + var runner = new MongoMigrationRunner( + database, + new IMongoMigration[] { migration }, + NullLogger.Instance, + TimeProvider.System); + + await runner.RunAsync(CancellationToken.None); + + var indexes = await database.GetCollection("documents.files").Indexes.ListAsync(); + var indexList = await indexes.ToListAsync(); + + var ttlIndex = indexList.Single(x => x["name"].AsString == "gridfs_files_expiresAt_ttl"); + Assert.Equal(0, ttlIndex["expireAfterSeconds"].ToDouble()); + } + finally + { + await _fixture.Client.DropDatabaseAsync(databaseName); + } + } + + [Fact] + public async Task EnsureGridFsExpiryIndexesMigration_DropsTtlIndexWhenRetentionDisabled() + { + var databaseName = $"feedser-gridfs-notl-{Guid.NewGuid():N}"; + var database = _fixture.Client.GetDatabase(databaseName); + await database.CreateCollectionAsync("documents.files"); + await database.CreateCollectionAsync(MongoStorageDefaults.Collections.Migrations); + + try + { + var collection = database.GetCollection("documents.files"); + var keys = Builders.IndexKeys.Ascending("metadata.expiresAt"); + var options = new CreateIndexOptions + { + Name = "gridfs_files_expiresAt_ttl", + ExpireAfter = TimeSpan.Zero, + PartialFilterExpression = Builders.Filter.Exists("metadata.expiresAt", true), + }; + + await collection.Indexes.CreateOneAsync(new CreateIndexModel(keys, options)); + + var migration = new EnsureGridFsExpiryIndexesMigration(Options.Create(new MongoStorageOptions + { + RawDocumentRetention = TimeSpan.Zero, + })); + + var runner = new MongoMigrationRunner( + database, + new IMongoMigration[] { migration }, + NullLogger.Instance, + TimeProvider.System); + + await runner.RunAsync(CancellationToken.None); + + var indexes = await collection.Indexes.ListAsync(); + var indexList = await indexes.ToListAsync(); + + Assert.DoesNotContain(indexList, x => x["name"].AsString == "gridfs_files_expiresAt_ttl"); + } + finally + { + await _fixture.Client.DropDatabaseAsync(databaseName); + } + } + + private sealed class TestMigration : IMongoMigration + { + public int ApplyCount { get; private set; } + + public string Id => "999_test"; + + public string Description => "test migration"; + + public Task ApplyAsync(IMongoDatabase database, CancellationToken cancellationToken) + { + ApplyCount++; + return Task.CompletedTask; + } + } +} diff --git a/src/StellaOps.Feedser.Storage.Mongo.Tests/MongoJobStoreTests.cs b/src/StellaOps.Feedser.Storage.Mongo.Tests/MongoJobStoreTests.cs new file mode 100644 index 00000000..1aea3b9c --- /dev/null +++ b/src/StellaOps.Feedser.Storage.Mongo.Tests/MongoJobStoreTests.cs @@ -0,0 +1,113 @@ +using Microsoft.Extensions.Logging.Abstractions; +using MongoDB.Driver; +using StellaOps.Feedser.Core.Jobs; +using StellaOps.Feedser.Storage.Mongo; + +namespace StellaOps.Feedser.Storage.Mongo.Tests; + +[Collection("mongo-fixture")] +public sealed class MongoJobStoreTests : IClassFixture +{ + private readonly MongoIntegrationFixture _fixture; + + public MongoJobStoreTests(MongoIntegrationFixture fixture) + { + _fixture = fixture; + } + + [Fact] + public async Task CreateStartCompleteLifecycle() + { + await ResetCollectionAsync(); + var collection = _fixture.Database.GetCollection(MongoStorageDefaults.Collections.Jobs); + var store = new MongoJobStore(collection, NullLogger.Instance); + + var request = new JobRunCreateRequest( + Kind: "mongo:test", + Trigger: "unit", + Parameters: new Dictionary { ["scope"] = "lifecycle" }, + ParametersHash: "abc", + Timeout: TimeSpan.FromSeconds(5), + LeaseDuration: TimeSpan.FromSeconds(2), + CreatedAt: DateTimeOffset.UtcNow); + + var created = await store.CreateAsync(request, CancellationToken.None); + Assert.Equal(JobRunStatus.Pending, created.Status); + + var started = await store.TryStartAsync(created.RunId, DateTimeOffset.UtcNow, CancellationToken.None); + Assert.NotNull(started); + Assert.Equal(JobRunStatus.Running, started!.Status); + + var completed = await store.TryCompleteAsync(created.RunId, new JobRunCompletion(JobRunStatus.Succeeded, DateTimeOffset.UtcNow, null), CancellationToken.None); + Assert.NotNull(completed); + Assert.Equal(JobRunStatus.Succeeded, completed!.Status); + + var recent = await store.GetRecentRunsAsync("mongo:test", 10, CancellationToken.None); + var snapshot = Assert.Single(recent); + Assert.Equal(JobRunStatus.Succeeded, snapshot.Status); + + var active = await store.GetActiveRunsAsync(CancellationToken.None); + Assert.Empty(active); + + var last = await store.GetLastRunAsync("mongo:test", CancellationToken.None); + Assert.NotNull(last); + Assert.Equal(completed.RunId, last!.RunId); + } + + [Fact] + public async Task StartAndFailRunHonorsStateTransitions() + { + await ResetCollectionAsync(); + var collection = _fixture.Database.GetCollection(MongoStorageDefaults.Collections.Jobs); + var store = new MongoJobStore(collection, NullLogger.Instance); + + var request = new JobRunCreateRequest( + Kind: "mongo:failure", + Trigger: "unit", + Parameters: new Dictionary(), + ParametersHash: null, + Timeout: null, + LeaseDuration: null, + CreatedAt: DateTimeOffset.UtcNow); + + var created = await store.CreateAsync(request, CancellationToken.None); + var firstStart = await store.TryStartAsync(created.RunId, DateTimeOffset.UtcNow, CancellationToken.None); + Assert.NotNull(firstStart); + + // Second start attempt should be rejected once running. + var secondStart = await store.TryStartAsync(created.RunId, DateTimeOffset.UtcNow.AddSeconds(1), CancellationToken.None); + Assert.Null(secondStart); + + var failure = await store.TryCompleteAsync( + created.RunId, + new JobRunCompletion(JobRunStatus.Failed, DateTimeOffset.UtcNow.AddSeconds(2), "boom"), + CancellationToken.None); + + Assert.NotNull(failure); + Assert.Equal("boom", failure!.Error); + Assert.Equal(JobRunStatus.Failed, failure.Status); + } + + [Fact] + public async Task CompletingUnknownRunReturnsNull() + { + await ResetCollectionAsync(); + var collection = _fixture.Database.GetCollection(MongoStorageDefaults.Collections.Jobs); + var store = new MongoJobStore(collection, NullLogger.Instance); + + var result = await store.TryCompleteAsync(Guid.NewGuid(), new JobRunCompletion(JobRunStatus.Succeeded, DateTimeOffset.UtcNow, null), CancellationToken.None); + + Assert.Null(result); + } + + private async Task ResetCollectionAsync() + { + try + { + await _fixture.Database.DropCollectionAsync(MongoStorageDefaults.Collections.Jobs); + } + catch (MongoCommandException ex) when (ex.CodeName == "NamespaceNotFound" || ex.Message.Contains("ns not found", StringComparison.OrdinalIgnoreCase)) + { + } + } +} diff --git a/src/StellaOps.Feedser.Storage.Mongo.Tests/MongoSourceStateRepositoryTests.cs b/src/StellaOps.Feedser.Storage.Mongo.Tests/MongoSourceStateRepositoryTests.cs new file mode 100644 index 00000000..af24393a --- /dev/null +++ b/src/StellaOps.Feedser.Storage.Mongo.Tests/MongoSourceStateRepositoryTests.cs @@ -0,0 +1,55 @@ +using Microsoft.Extensions.Logging.Abstractions; +using MongoDB.Bson; +using StellaOps.Feedser.Storage.Mongo; + +namespace StellaOps.Feedser.Storage.Mongo.Tests; + +[Collection("mongo-fixture")] +public sealed class MongoSourceStateRepositoryTests : IClassFixture +{ + private readonly MongoIntegrationFixture _fixture; + + public MongoSourceStateRepositoryTests(MongoIntegrationFixture fixture) + { + _fixture = fixture; + } + + [Fact] + public async Task UpsertAndUpdateCursorFlow() + { + var repository = new MongoSourceStateRepository(_fixture.Database, NullLogger.Instance); + var sourceName = "nvd"; + + var record = new SourceStateRecord( + SourceName: sourceName, + Enabled: true, + Paused: false, + Cursor: new BsonDocument("page", 1), + LastSuccess: null, + LastFailure: null, + FailCount: 0, + BackoffUntil: null, + UpdatedAt: DateTimeOffset.UtcNow, + LastFailureReason: null); + + var upserted = await repository.UpsertAsync(record, CancellationToken.None); + Assert.True(upserted.Enabled); + + var cursor = new BsonDocument("page", 2); + var updated = await repository.UpdateCursorAsync(sourceName, cursor, DateTimeOffset.UtcNow, CancellationToken.None); + Assert.NotNull(updated); + Assert.Equal(0, updated!.FailCount); + Assert.Equal(2, updated.Cursor["page"].AsInt32); + + var failure = await repository.MarkFailureAsync(sourceName, DateTimeOffset.UtcNow, TimeSpan.FromMinutes(5), "network timeout", CancellationToken.None); + Assert.NotNull(failure); + Assert.Equal(1, failure!.FailCount); + Assert.NotNull(failure.BackoffUntil); + Assert.Equal("network timeout", failure.LastFailureReason); + + var fetched = await repository.TryGetAsync(sourceName, CancellationToken.None); + Assert.NotNull(fetched); + Assert.Equal(failure.BackoffUntil, fetched!.BackoffUntil); + Assert.Equal("network timeout", fetched.LastFailureReason); + } +} diff --git a/src/StellaOps.Feedser.Storage.Mongo.Tests/RawDocumentRetentionServiceTests.cs b/src/StellaOps.Feedser.Storage.Mongo.Tests/RawDocumentRetentionServiceTests.cs new file mode 100644 index 00000000..8112b16b --- /dev/null +++ b/src/StellaOps.Feedser.Storage.Mongo.Tests/RawDocumentRetentionServiceTests.cs @@ -0,0 +1,93 @@ +using Microsoft.Extensions.Logging.Abstractions; +using Microsoft.Extensions.Options; +using Microsoft.Extensions.Time.Testing; +using MongoDB.Bson; +using MongoDB.Driver; +using MongoDB.Driver.GridFS; +using StellaOps.Feedser.Storage.Mongo; +using StellaOps.Feedser.Storage.Mongo.Documents; +using StellaOps.Feedser.Storage.Mongo.Dtos; + +namespace StellaOps.Feedser.Storage.Mongo.Tests; + +[Collection("mongo-fixture")] +public sealed class RawDocumentRetentionServiceTests : IClassFixture +{ + private readonly MongoIntegrationFixture _fixture; + + public RawDocumentRetentionServiceTests(MongoIntegrationFixture fixture) + { + _fixture = fixture; + } + + [Fact] + public async Task SweepExpiredDocumentsAsync_RemovesExpiredRawDocuments() + { + var database = _fixture.Database; + var documents = database.GetCollection(MongoStorageDefaults.Collections.Document); + var dtos = database.GetCollection(MongoStorageDefaults.Collections.Dto); + var bucket = new GridFSBucket(database, new GridFSBucketOptions { BucketName = "documents" }); + + var now = new DateTimeOffset(2024, 10, 1, 12, 0, 0, TimeSpan.Zero); + var fakeTime = new FakeTimeProvider(now); + + var options = Options.Create(new MongoStorageOptions + { + ConnectionString = _fixture.Runner.ConnectionString, + DatabaseName = database.DatabaseNamespace.DatabaseName, + RawDocumentRetention = TimeSpan.FromDays(1), + RawDocumentRetentionTtlGrace = TimeSpan.Zero, + RawDocumentRetentionSweepInterval = TimeSpan.FromMinutes(5), + }); + + var expiredId = Guid.NewGuid().ToString(); + var gridFsId = await bucket.UploadFromBytesAsync("expired", new byte[] { 1, 2, 3 }); + await documents.InsertOneAsync(new DocumentDocument + { + Id = expiredId, + SourceName = "nvd", + Uri = "https://example.test/cve", + FetchedAt = now.AddDays(-2).UtcDateTime, + Sha256 = "abc", + Status = "pending", + ExpiresAt = now.AddMinutes(-5).UtcDateTime, + GridFsId = gridFsId, + }); + + await dtos.InsertOneAsync(new DtoDocument + { + Id = Guid.NewGuid().ToString(), + DocumentId = expiredId, + SourceName = "nvd", + SchemaVersion = "schema", + Payload = new BsonDocument("value", 1), + ValidatedAt = now.UtcDateTime, + }); + + var freshId = Guid.NewGuid().ToString(); + await documents.InsertOneAsync(new DocumentDocument + { + Id = freshId, + SourceName = "nvd", + Uri = "https://example.test/future", + FetchedAt = now.UtcDateTime, + Sha256 = "def", + Status = "pending", + ExpiresAt = now.AddHours(1).UtcDateTime, + GridFsId = null, + }); + + var service = new RawDocumentRetentionService(database, options, NullLogger.Instance, fakeTime); + + var removed = await service.SweepExpiredDocumentsAsync(CancellationToken.None); + + Assert.Equal(1, removed); + Assert.Equal(0, await documents.CountDocumentsAsync(d => d.Id == expiredId)); + Assert.Equal(0, await dtos.CountDocumentsAsync(d => d.DocumentId == expiredId)); + Assert.Equal(1, await documents.CountDocumentsAsync(d => d.Id == freshId)); + + var filter = Builders.Filter.Eq("_id", gridFsId); + using var cursor = await bucket.FindAsync(filter); + Assert.Empty(await cursor.ToListAsync()); + } +} diff --git a/src/StellaOps.Feedser.Storage.Mongo.Tests/StellaOps.Feedser.Storage.Mongo.Tests.csproj b/src/StellaOps.Feedser.Storage.Mongo.Tests/StellaOps.Feedser.Storage.Mongo.Tests.csproj new file mode 100644 index 00000000..910f1f4e --- /dev/null +++ b/src/StellaOps.Feedser.Storage.Mongo.Tests/StellaOps.Feedser.Storage.Mongo.Tests.csproj @@ -0,0 +1,12 @@ + + + net10.0 + enable + enable + + + + + + + diff --git a/src/StellaOps.Feedser.Storage.Mongo/AGENTS.md b/src/StellaOps.Feedser.Storage.Mongo/AGENTS.md new file mode 100644 index 00000000..a1f2b543 --- /dev/null +++ b/src/StellaOps.Feedser.Storage.Mongo/AGENTS.md @@ -0,0 +1,29 @@ +# AGENTS +## Role +Canonical persistence for raw documents, DTOs, canonical advisories, jobs, and state. Provides repositories and bootstrapper for collections/indexes. +## Scope +- Collections (MongoStorageDefaults): source, source_state, document, dto, advisory, alias, affected, reference, kev_flag, ru_flags, jp_flags, psirt_flags, merge_event, export_state, locks, jobs; GridFS bucket fs.documents; field names include ttlAt (locks), sourceName, uri, advisoryKey. +- Records: SourceState (cursor, lastSuccess/error, failCount, backoffUntil), JobRun, MergeEvent, ExportState, Advisory documents mirroring Models with embedded arrays when practical. +- Bootstrapper: create collections, indexes (unique advisoryKey, scheme/value, platform/name, published, modified), TTL on locks, and validate connectivity for /ready health probes. +- Job store: create, read, mark completed/failed; compute durations; recent/last queries; active by status. +- Advisory store: CRUD for canonical advisories; query by key/alias and list for exporters with deterministic paging. +## Participants +- Core jobs read/write runs and leases; WebService /ready pings database; /jobs APIs query runs/definitions. +- Source connectors store raw docs, DTOs, and mapped canonical advisories with provenance; Update SourceState cursor/backoff. +- Exporters read advisories and write export_state. +## Interfaces & contracts +- IMongoDatabase injected; MongoUrl from options; database name from options or MongoUrl or default "feedser". +- Repositories expose async methods with CancellationToken; deterministic sorting. +- All date/time values stored as UTC; identifiers normalized. +## In/Out of scope +In: persistence, bootstrap, indexes, basic query helpers. +Out: business mapping logic, HTTP, packaging. +## Observability & security expectations +- Log collection/index creation; warn on existing mismatches. +- Timeouts and retry policies; avoid unbounded scans; page reads. +- Do not log DSNs with credentials; redact in diagnostics. +## Tests +- Author and review coverage in `../StellaOps.Feedser.Storage.Mongo.Tests`. +- Shared fixtures (e.g., `MongoIntegrationFixture`, `ConnectorTestHarness`) live in `../StellaOps.Feedser.Testing`. +- Keep fixtures deterministic; match new cases to real-world advisories or regression scenarios. + diff --git a/src/StellaOps.Feedser.Storage.Mongo/Advisories/AdvisoryDocument.cs b/src/StellaOps.Feedser.Storage.Mongo/Advisories/AdvisoryDocument.cs new file mode 100644 index 00000000..8077974e --- /dev/null +++ b/src/StellaOps.Feedser.Storage.Mongo/Advisories/AdvisoryDocument.cs @@ -0,0 +1,32 @@ +using System.Collections.Generic; +using MongoDB.Bson; +using MongoDB.Bson.Serialization.Attributes; + +namespace StellaOps.Feedser.Storage.Mongo.Advisories; + +[BsonIgnoreExtraElements] +public sealed class AdvisoryDocument +{ + [BsonId] + public string Id { get; set; } = string.Empty; + + [BsonElement("advisoryKey")] + public string AdvisoryKey + { + get => Id; + set => Id = value; + } + + [BsonElement("payload")] + public BsonDocument Payload { get; set; } = new(); + + [BsonElement("modified")] + public DateTime Modified { get; set; } + + [BsonElement("published")] + public DateTime? Published { get; set; } + + [BsonElement("normalizedVersions")] + [BsonIgnoreIfNull] + public List? NormalizedVersions { get; set; } +} diff --git a/src/StellaOps.Feedser.Storage.Mongo/Advisories/AdvisoryStore.cs b/src/StellaOps.Feedser.Storage.Mongo/Advisories/AdvisoryStore.cs new file mode 100644 index 00000000..fe685339 --- /dev/null +++ b/src/StellaOps.Feedser.Storage.Mongo/Advisories/AdvisoryStore.cs @@ -0,0 +1,461 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Runtime.CompilerServices; +using System.Text.Json; +using System.Text.Json.Serialization; +using Microsoft.Extensions.Logging; +using MongoDB.Bson; +using MongoDB.Driver; +using StellaOps.Feedser.Models; +using StellaOps.Feedser.Storage.Mongo.Aliases; + +namespace StellaOps.Feedser.Storage.Mongo.Advisories; + +public sealed class AdvisoryStore : IAdvisoryStore +{ + private readonly IMongoCollection _collection; + private readonly ILogger _logger; + private readonly IAliasStore _aliasStore; + private readonly TimeProvider _timeProvider; + + public AdvisoryStore( + IMongoDatabase database, + IAliasStore aliasStore, + ILogger logger, + TimeProvider? timeProvider = null) + { + _collection = (database ?? throw new ArgumentNullException(nameof(database))) + .GetCollection(MongoStorageDefaults.Collections.Advisory); + _aliasStore = aliasStore ?? throw new ArgumentNullException(nameof(aliasStore)); + _logger = logger ?? throw new ArgumentNullException(nameof(logger)); + _timeProvider = timeProvider ?? TimeProvider.System; + } + + + public async Task UpsertAsync(Advisory advisory, CancellationToken cancellationToken) + { + ArgumentNullException.ThrowIfNull(advisory); + + var missing = ProvenanceInspector.FindMissingProvenance(advisory); + var primarySource = advisory.Provenance.FirstOrDefault()?.Source ?? "unknown"; + foreach (var item in missing) + { + var source = string.IsNullOrWhiteSpace(item.Source) ? primarySource : item.Source; + _logger.LogWarning( + "Missing provenance detected for {Component} in advisory {AdvisoryKey} (source {Source}).", + item.Component, + advisory.AdvisoryKey, + source); + ProvenanceDiagnostics.RecordMissing(source, item.Component, item.RecordedAt, item.FieldMask); + } + + var payload = CanonicalJsonSerializer.Serialize(advisory); + var normalizedVersions = NormalizedVersionDocumentFactory.Create(advisory); + var document = new AdvisoryDocument + { + AdvisoryKey = advisory.AdvisoryKey, + Payload = BsonDocument.Parse(payload), + Modified = advisory.Modified?.UtcDateTime ?? DateTime.UtcNow, + Published = advisory.Published?.UtcDateTime, + NormalizedVersions = normalizedVersions, + }; + + var options = new ReplaceOptions { IsUpsert = true }; + await _collection.ReplaceOneAsync(x => x.AdvisoryKey == advisory.AdvisoryKey, document, options, cancellationToken).ConfigureAwait(false); + _logger.LogDebug("Upserted advisory {AdvisoryKey}", advisory.AdvisoryKey); + + var aliasEntries = BuildAliasEntries(advisory); + var updatedAt = _timeProvider.GetUtcNow(); + await _aliasStore.ReplaceAsync(advisory.AdvisoryKey, aliasEntries, updatedAt, cancellationToken).ConfigureAwait(false); + } + + public async Task FindAsync(string advisoryKey, CancellationToken cancellationToken) + { + ArgumentException.ThrowIfNullOrEmpty(advisoryKey); + var document = await _collection.Find(x => x.AdvisoryKey == advisoryKey) + .FirstOrDefaultAsync(cancellationToken) + .ConfigureAwait(false); + + return document is null ? null : Deserialize(document.Payload); + } + + private static IEnumerable BuildAliasEntries(Advisory advisory) + { + foreach (var alias in advisory.Aliases) + { + if (AliasSchemeRegistry.TryGetScheme(alias, out var scheme)) + { + yield return new AliasEntry(scheme, alias); + } + else + { + yield return new AliasEntry(AliasStoreConstants.UnscopedScheme, alias); + } + } + + yield return new AliasEntry(AliasStoreConstants.PrimaryScheme, advisory.AdvisoryKey); + } + + public async Task> GetRecentAsync(int limit, CancellationToken cancellationToken) + { + var cursor = await _collection.Find(FilterDefinition.Empty) + .SortByDescending(x => x.Modified) + .Limit(limit) + .ToListAsync(cancellationToken) + .ConfigureAwait(false); + + return cursor.Select(static doc => Deserialize(doc.Payload)).ToArray(); + } + + public async IAsyncEnumerable StreamAsync([EnumeratorCancellation] CancellationToken cancellationToken) + { + var options = new FindOptions + { + Sort = Builders.Sort.Ascending(static doc => doc.AdvisoryKey), + }; + + using var cursor = await _collection.FindAsync( + FilterDefinition.Empty, + options, + cancellationToken) + .ConfigureAwait(false); + + while (await cursor.MoveNextAsync(cancellationToken).ConfigureAwait(false)) + { + foreach (var document in cursor.Current) + { + cancellationToken.ThrowIfCancellationRequested(); + yield return Deserialize(document.Payload); + } + } + } + + private static Advisory Deserialize(BsonDocument payload) + { + ArgumentNullException.ThrowIfNull(payload); + + var advisoryKey = payload.GetValue("advisoryKey", defaultValue: null)?.AsString + ?? throw new InvalidOperationException("advisoryKey missing from payload."); + var title = payload.GetValue("title", defaultValue: null)?.AsString ?? advisoryKey; + + string? summary = payload.TryGetValue("summary", out var summaryValue) && summaryValue.IsString ? summaryValue.AsString : null; + string? language = payload.TryGetValue("language", out var languageValue) && languageValue.IsString ? languageValue.AsString : null; + DateTimeOffset? published = TryReadDateTime(payload, "published"); + DateTimeOffset? modified = TryReadDateTime(payload, "modified"); + string? severity = payload.TryGetValue("severity", out var severityValue) && severityValue.IsString ? severityValue.AsString : null; + var exploitKnown = payload.TryGetValue("exploitKnown", out var exploitValue) && exploitValue.IsBoolean && exploitValue.AsBoolean; + + var aliases = payload.TryGetValue("aliases", out var aliasValue) && aliasValue is BsonArray aliasArray + ? aliasArray.OfType().Where(static x => x.IsString).Select(static x => x.AsString) + : Array.Empty(); + + var credits = payload.TryGetValue("credits", out var creditsValue) && creditsValue is BsonArray creditsArray + ? creditsArray.OfType().Select(DeserializeCredit).ToArray() + : Array.Empty(); + + var references = payload.TryGetValue("references", out var referencesValue) && referencesValue is BsonArray referencesArray + ? referencesArray.OfType().Select(DeserializeReference).ToArray() + : Array.Empty(); + + var affectedPackages = payload.TryGetValue("affectedPackages", out var affectedValue) && affectedValue is BsonArray affectedArray + ? affectedArray.OfType().Select(DeserializeAffectedPackage).ToArray() + : Array.Empty(); + + var cvssMetrics = payload.TryGetValue("cvssMetrics", out var cvssValue) && cvssValue is BsonArray cvssArray + ? cvssArray.OfType().Select(DeserializeCvssMetric).ToArray() + : Array.Empty(); + + var provenance = payload.TryGetValue("provenance", out var provenanceValue) && provenanceValue is BsonArray provenanceArray + ? provenanceArray.OfType().Select(DeserializeProvenance).ToArray() + : Array.Empty(); + + return new Advisory( + advisoryKey, + title, + summary, + language, + published, + modified, + severity, + exploitKnown, + aliases, + credits, + references, + affectedPackages, + cvssMetrics, + provenance); + } + + private static AdvisoryReference DeserializeReference(BsonDocument document) + { + var url = document.GetValue("url", defaultValue: null)?.AsString + ?? throw new InvalidOperationException("reference.url missing from payload."); + string? kind = document.TryGetValue("kind", out var kindValue) && kindValue.IsString ? kindValue.AsString : null; + string? sourceTag = document.TryGetValue("sourceTag", out var sourceTagValue) && sourceTagValue.IsString ? sourceTagValue.AsString : null; + string? summary = document.TryGetValue("summary", out var summaryValue) && summaryValue.IsString ? summaryValue.AsString : null; + var provenance = document.TryGetValue("provenance", out var provenanceValue) && provenanceValue.IsBsonDocument + ? DeserializeProvenance(provenanceValue.AsBsonDocument) + : AdvisoryProvenance.Empty; + + return new AdvisoryReference(url, kind, sourceTag, summary, provenance); + } + + private static AdvisoryCredit DeserializeCredit(BsonDocument document) + { + var displayName = document.GetValue("displayName", defaultValue: null)?.AsString + ?? throw new InvalidOperationException("credits.displayName missing from payload."); + string? role = document.TryGetValue("role", out var roleValue) && roleValue.IsString ? roleValue.AsString : null; + var contacts = document.TryGetValue("contacts", out var contactsValue) && contactsValue is BsonArray contactsArray + ? contactsArray.OfType().Where(static value => value.IsString).Select(static value => value.AsString).ToArray() + : Array.Empty(); + var provenance = document.TryGetValue("provenance", out var provenanceValue) && provenanceValue.IsBsonDocument + ? DeserializeProvenance(provenanceValue.AsBsonDocument) + : AdvisoryProvenance.Empty; + + return new AdvisoryCredit(displayName, role, contacts, provenance); + } + + private static AffectedPackage DeserializeAffectedPackage(BsonDocument document) + { + var type = document.GetValue("type", defaultValue: null)?.AsString + ?? throw new InvalidOperationException("affectedPackages.type missing from payload."); + var identifier = document.GetValue("identifier", defaultValue: null)?.AsString + ?? throw new InvalidOperationException("affectedPackages.identifier missing from payload."); + string? platform = document.TryGetValue("platform", out var platformValue) && platformValue.IsString ? platformValue.AsString : null; + + var versionRanges = document.TryGetValue("versionRanges", out var rangesValue) && rangesValue is BsonArray rangesArray + ? rangesArray.OfType().Select(DeserializeVersionRange).ToArray() + : Array.Empty(); + + var statuses = document.TryGetValue("statuses", out var statusesValue) && statusesValue is BsonArray statusesArray + ? statusesArray.OfType().Select(DeserializeStatus).ToArray() + : Array.Empty(); + + var normalizedVersions = document.TryGetValue("normalizedVersions", out var normalizedValue) && normalizedValue is BsonArray normalizedArray + ? normalizedArray.OfType().Select(DeserializeNormalizedVersionRule).ToArray() + : Array.Empty(); + + var provenance = document.TryGetValue("provenance", out var provenanceValue) && provenanceValue is BsonArray provenanceArray + ? provenanceArray.OfType().Select(DeserializeProvenance).ToArray() + : Array.Empty(); + + return new AffectedPackage(type, identifier, platform, versionRanges, statuses, provenance, normalizedVersions); + } + + private static AffectedVersionRange DeserializeVersionRange(BsonDocument document) + { + var rangeKind = document.GetValue("rangeKind", defaultValue: null)?.AsString + ?? throw new InvalidOperationException("versionRanges.rangeKind missing from payload."); + string? introducedVersion = document.TryGetValue("introducedVersion", out var introducedValue) && introducedValue.IsString ? introducedValue.AsString : null; + string? fixedVersion = document.TryGetValue("fixedVersion", out var fixedValue) && fixedValue.IsString ? fixedValue.AsString : null; + string? lastAffectedVersion = document.TryGetValue("lastAffectedVersion", out var lastValue) && lastValue.IsString ? lastValue.AsString : null; + string? rangeExpression = document.TryGetValue("rangeExpression", out var expressionValue) && expressionValue.IsString ? expressionValue.AsString : null; + var provenance = document.TryGetValue("provenance", out var provenanceValue) && provenanceValue.IsBsonDocument + ? DeserializeProvenance(provenanceValue.AsBsonDocument) + : AdvisoryProvenance.Empty; + RangePrimitives? primitives = null; + if (document.TryGetValue("primitives", out var primitivesValue) && primitivesValue.IsBsonDocument) + { + primitives = DeserializePrimitives(primitivesValue.AsBsonDocument); + } + + return new AffectedVersionRange(rangeKind, introducedVersion, fixedVersion, lastAffectedVersion, rangeExpression, provenance, primitives); + } + + private static AffectedPackageStatus DeserializeStatus(BsonDocument document) + { + var status = document.GetValue("status", defaultValue: null)?.AsString + ?? throw new InvalidOperationException("statuses.status missing from payload."); + var provenance = document.TryGetValue("provenance", out var provenanceValue) && provenanceValue.IsBsonDocument + ? DeserializeProvenance(provenanceValue.AsBsonDocument) + : AdvisoryProvenance.Empty; + + return new AffectedPackageStatus(status, provenance); + } + + private static CvssMetric DeserializeCvssMetric(BsonDocument document) + { + var version = document.GetValue("version", defaultValue: null)?.AsString + ?? throw new InvalidOperationException("cvssMetrics.version missing from payload."); + var vector = document.GetValue("vector", defaultValue: null)?.AsString + ?? throw new InvalidOperationException("cvssMetrics.vector missing from payload."); + var baseScore = document.TryGetValue("baseScore", out var scoreValue) && scoreValue.IsNumeric ? scoreValue.ToDouble() : 0d; + var baseSeverity = document.GetValue("baseSeverity", defaultValue: null)?.AsString + ?? throw new InvalidOperationException("cvssMetrics.baseSeverity missing from payload."); + var provenance = document.TryGetValue("provenance", out var provenanceValue) && provenanceValue.IsBsonDocument + ? DeserializeProvenance(provenanceValue.AsBsonDocument) + : AdvisoryProvenance.Empty; + + return new CvssMetric(version, vector, baseScore, baseSeverity, provenance); + } + + private static AdvisoryProvenance DeserializeProvenance(BsonDocument document) + { + var source = document.GetValue("source", defaultValue: null)?.AsString + ?? throw new InvalidOperationException("provenance.source missing from payload."); + var kind = document.GetValue("kind", defaultValue: null)?.AsString + ?? throw new InvalidOperationException("provenance.kind missing from payload."); + string? value = document.TryGetValue("value", out var valueElement) && valueElement.IsString ? valueElement.AsString : null; + string? decisionReason = document.TryGetValue("decisionReason", out var reasonElement) && reasonElement.IsString ? reasonElement.AsString : null; + var recordedAt = TryConvertDateTime(document.GetValue("recordedAt", defaultValue: null)); + IEnumerable? fieldMask = null; + if (document.TryGetValue("fieldMask", out var fieldMaskValue) && fieldMaskValue is BsonArray fieldMaskArray) + { + fieldMask = fieldMaskArray + .OfType() + .Where(static element => element.IsString) + .Select(static element => element.AsString); + } + + return new AdvisoryProvenance( + source, + kind, + value ?? string.Empty, + recordedAt ?? DateTimeOffset.UtcNow, + fieldMask, + decisionReason); + } + + private static NormalizedVersionRule DeserializeNormalizedVersionRule(BsonDocument document) + { + var scheme = document.GetValue("scheme", defaultValue: null)?.AsString + ?? throw new InvalidOperationException("normalizedVersions.scheme missing from payload."); + var type = document.GetValue("type", defaultValue: null)?.AsString + ?? throw new InvalidOperationException("normalizedVersions.type missing from payload."); + + string? min = document.TryGetValue("min", out var minValue) && minValue.IsString ? minValue.AsString : null; + bool? minInclusive = document.TryGetValue("minInclusive", out var minInclusiveValue) && minInclusiveValue.IsBoolean ? minInclusiveValue.AsBoolean : null; + string? max = document.TryGetValue("max", out var maxValue) && maxValue.IsString ? maxValue.AsString : null; + bool? maxInclusive = document.TryGetValue("maxInclusive", out var maxInclusiveValue) && maxInclusiveValue.IsBoolean ? maxInclusiveValue.AsBoolean : null; + string? value = document.TryGetValue("value", out var valueElement) && valueElement.IsString ? valueElement.AsString : null; + string? notes = document.TryGetValue("notes", out var notesValue) && notesValue.IsString ? notesValue.AsString : null; + + return new NormalizedVersionRule( + scheme, + type, + min, + minInclusive, + max, + maxInclusive, + value, + notes); + } + + private static RangePrimitives? DeserializePrimitives(BsonDocument document) + { + SemVerPrimitive? semVer = null; + NevraPrimitive? nevra = null; + EvrPrimitive? evr = null; + IReadOnlyDictionary? vendor = null; + + if (document.TryGetValue("semVer", out var semverValue) && semverValue.IsBsonDocument) + { + var semverDoc = semverValue.AsBsonDocument; + semVer = new SemVerPrimitive( + semverDoc.TryGetValue("introduced", out var semIntroduced) && semIntroduced.IsString ? semIntroduced.AsString : null, + semverDoc.TryGetValue("introducedInclusive", out var semIntroducedInclusive) && semIntroducedInclusive.IsBoolean && semIntroducedInclusive.AsBoolean, + semverDoc.TryGetValue("fixed", out var semFixed) && semFixed.IsString ? semFixed.AsString : null, + semverDoc.TryGetValue("fixedInclusive", out var semFixedInclusive) && semFixedInclusive.IsBoolean && semFixedInclusive.AsBoolean, + semverDoc.TryGetValue("lastAffected", out var semLast) && semLast.IsString ? semLast.AsString : null, + semverDoc.TryGetValue("lastAffectedInclusive", out var semLastInclusive) && semLastInclusive.IsBoolean && semLastInclusive.AsBoolean, + semverDoc.TryGetValue("constraintExpression", out var constraint) && constraint.IsString ? constraint.AsString : null); + } + + if (document.TryGetValue("nevra", out var nevraValue) && nevraValue.IsBsonDocument) + { + var nevraDoc = nevraValue.AsBsonDocument; + nevra = new NevraPrimitive( + DeserializeNevraComponent(nevraDoc, "introduced"), + DeserializeNevraComponent(nevraDoc, "fixed"), + DeserializeNevraComponent(nevraDoc, "lastAffected")); + } + + if (document.TryGetValue("evr", out var evrValue) && evrValue.IsBsonDocument) + { + var evrDoc = evrValue.AsBsonDocument; + evr = new EvrPrimitive( + DeserializeEvrComponent(evrDoc, "introduced"), + DeserializeEvrComponent(evrDoc, "fixed"), + DeserializeEvrComponent(evrDoc, "lastAffected")); + } + + if (document.TryGetValue("vendorExtensions", out var vendorValue) && vendorValue.IsBsonDocument) + { + vendor = vendorValue.AsBsonDocument.Elements + .Where(static e => e.Value.IsString) + .ToDictionary(static e => e.Name, static e => e.Value.AsString, StringComparer.Ordinal); + if (vendor.Count == 0) + { + vendor = null; + } + } + + if (semVer is null && nevra is null && evr is null && vendor is null) + { + return null; + } + + return new RangePrimitives(semVer, nevra, evr, vendor); + } + + private static NevraComponent? DeserializeNevraComponent(BsonDocument parent, string field) + { + if (!parent.TryGetValue(field, out var value) || !value.IsBsonDocument) + { + return null; + } + + var component = value.AsBsonDocument; + var name = component.TryGetValue("name", out var nameValue) && nameValue.IsString ? nameValue.AsString : null; + var version = component.TryGetValue("version", out var versionValue) && versionValue.IsString ? versionValue.AsString : null; + if (name is null || version is null) + { + return null; + } + + var epoch = component.TryGetValue("epoch", out var epochValue) && epochValue.IsNumeric ? epochValue.ToInt32() : 0; + var release = component.TryGetValue("release", out var releaseValue) && releaseValue.IsString ? releaseValue.AsString : string.Empty; + var architecture = component.TryGetValue("architecture", out var archValue) && archValue.IsString ? archValue.AsString : null; + + return new NevraComponent(name, epoch, version, release, architecture); + } + + private static EvrComponent? DeserializeEvrComponent(BsonDocument parent, string field) + { + if (!parent.TryGetValue(field, out var value) || !value.IsBsonDocument) + { + return null; + } + + var component = value.AsBsonDocument; + var epoch = component.TryGetValue("epoch", out var epochValue) && epochValue.IsNumeric ? epochValue.ToInt32() : 0; + var upstream = component.TryGetValue("upstreamVersion", out var upstreamValue) && upstreamValue.IsString ? upstreamValue.AsString : null; + if (upstream is null) + { + return null; + } + + var revision = component.TryGetValue("revision", out var revisionValue) && revisionValue.IsString ? revisionValue.AsString : null; + return new EvrComponent(epoch, upstream, revision); + } + + private static DateTimeOffset? TryReadDateTime(BsonDocument document, string field) + => document.TryGetValue(field, out var value) ? TryConvertDateTime(value) : null; + + private static DateTimeOffset? TryConvertDateTime(BsonValue? value) + { + if (value is null) + { + return null; + } + + return value switch + { + BsonDateTime dateTime => DateTime.SpecifyKind(dateTime.ToUniversalTime(), DateTimeKind.Utc), + BsonString stringValue when DateTimeOffset.TryParse(stringValue.AsString, out var parsed) => parsed.ToUniversalTime(), + _ => null, + }; + } + +} diff --git a/src/StellaOps.Feedser.Storage.Mongo/Advisories/IAdvisoryStore.cs b/src/StellaOps.Feedser.Storage.Mongo/Advisories/IAdvisoryStore.cs new file mode 100644 index 00000000..f7f3209b --- /dev/null +++ b/src/StellaOps.Feedser.Storage.Mongo/Advisories/IAdvisoryStore.cs @@ -0,0 +1,14 @@ +using StellaOps.Feedser.Models; + +namespace StellaOps.Feedser.Storage.Mongo.Advisories; + +public interface IAdvisoryStore +{ + Task UpsertAsync(Advisory advisory, CancellationToken cancellationToken); + + Task FindAsync(string advisoryKey, CancellationToken cancellationToken); + + Task> GetRecentAsync(int limit, CancellationToken cancellationToken); + + IAsyncEnumerable StreamAsync(CancellationToken cancellationToken); +} diff --git a/src/StellaOps.Feedser.Storage.Mongo/Advisories/NormalizedVersionDocument.cs b/src/StellaOps.Feedser.Storage.Mongo/Advisories/NormalizedVersionDocument.cs new file mode 100644 index 00000000..2d89e021 --- /dev/null +++ b/src/StellaOps.Feedser.Storage.Mongo/Advisories/NormalizedVersionDocument.cs @@ -0,0 +1,64 @@ +using System; +using MongoDB.Bson.Serialization.Attributes; + +namespace StellaOps.Feedser.Storage.Mongo.Advisories; + +[BsonIgnoreExtraElements] +public sealed class NormalizedVersionDocument +{ + [BsonElement("packageId")] + public string PackageId { get; set; } = string.Empty; + + [BsonElement("packageType")] + public string PackageType { get; set; } = string.Empty; + + [BsonElement("scheme")] + public string Scheme { get; set; } = string.Empty; + + [BsonElement("type")] + public string Type { get; set; } = string.Empty; + + [BsonElement("style")] + [BsonIgnoreIfNull] + public string? Style { get; set; } + + [BsonElement("min")] + [BsonIgnoreIfNull] + public string? Min { get; set; } + + [BsonElement("minInclusive")] + [BsonIgnoreIfNull] + public bool? MinInclusive { get; set; } + + [BsonElement("max")] + [BsonIgnoreIfNull] + public string? Max { get; set; } + + [BsonElement("maxInclusive")] + [BsonIgnoreIfNull] + public bool? MaxInclusive { get; set; } + + [BsonElement("value")] + [BsonIgnoreIfNull] + public string? Value { get; set; } + + [BsonElement("notes")] + [BsonIgnoreIfNull] + public string? Notes { get; set; } + + [BsonElement("decisionReason")] + [BsonIgnoreIfNull] + public string? DecisionReason { get; set; } + + [BsonElement("constraint")] + [BsonIgnoreIfNull] + public string? Constraint { get; set; } + + [BsonElement("source")] + [BsonIgnoreIfNull] + public string? Source { get; set; } + + [BsonElement("recordedAt")] + [BsonIgnoreIfNull] + public DateTime? RecordedAtUtc { get; set; } +} diff --git a/src/StellaOps.Feedser.Storage.Mongo/Advisories/NormalizedVersionDocumentFactory.cs b/src/StellaOps.Feedser.Storage.Mongo/Advisories/NormalizedVersionDocumentFactory.cs new file mode 100644 index 00000000..12cd060e --- /dev/null +++ b/src/StellaOps.Feedser.Storage.Mongo/Advisories/NormalizedVersionDocumentFactory.cs @@ -0,0 +1,100 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using StellaOps.Feedser.Models; + +namespace StellaOps.Feedser.Storage.Mongo.Advisories; + +internal static class NormalizedVersionDocumentFactory +{ + public static List? Create(Advisory advisory) + { + if (advisory.AffectedPackages.IsDefaultOrEmpty || advisory.AffectedPackages.Length == 0) + { + return null; + } + + var documents = new List(); + var advisoryFallbackReason = advisory.Provenance.FirstOrDefault()?.DecisionReason; + var advisoryFallbackSource = advisory.Provenance.FirstOrDefault()?.Source; + var advisoryFallbackRecordedAt = advisory.Provenance.FirstOrDefault()?.RecordedAt; + + foreach (var package in advisory.AffectedPackages) + { + if (package.NormalizedVersions.IsDefaultOrEmpty || package.NormalizedVersions.Length == 0) + { + continue; + } + + foreach (var rule in package.NormalizedVersions) + { + var matchingRange = FindMatchingRange(package, rule); + var decisionReason = matchingRange?.Provenance.DecisionReason + ?? package.Provenance.FirstOrDefault()?.DecisionReason + ?? advisoryFallbackReason; + + var source = matchingRange?.Provenance.Source + ?? package.Provenance.FirstOrDefault()?.Source + ?? advisoryFallbackSource; + + var recordedAt = matchingRange?.Provenance.RecordedAt + ?? package.Provenance.FirstOrDefault()?.RecordedAt + ?? advisoryFallbackRecordedAt; + + var constraint = matchingRange?.RangeExpression + ?? matchingRange?.Primitives?.SemVer?.ConstraintExpression; + + var style = rule.Type; + + documents.Add(new NormalizedVersionDocument + { + PackageId = package.Identifier ?? string.Empty, + PackageType = package.Type ?? string.Empty, + Scheme = rule.Scheme, + Type = rule.Type, + Style = style, + Min = rule.Min, + MinInclusive = rule.MinInclusive, + Max = rule.Max, + MaxInclusive = rule.MaxInclusive, + Value = rule.Value, + Notes = rule.Notes, + DecisionReason = decisionReason, + Constraint = constraint, + Source = source, + RecordedAtUtc = recordedAt?.UtcDateTime, + }); + } + } + + return documents.Count == 0 ? null : documents; + } + + private static AffectedVersionRange? FindMatchingRange(AffectedPackage package, NormalizedVersionRule rule) + { + foreach (var range in package.VersionRanges) + { + var candidate = range.ToNormalizedVersionRule(rule.Notes); + if (candidate is null) + { + continue; + } + + if (NormalizedRulesEquivalent(candidate, rule)) + { + return range; + } + } + + return null; + } + + private static bool NormalizedRulesEquivalent(NormalizedVersionRule left, NormalizedVersionRule right) + => string.Equals(left.Scheme, right.Scheme, StringComparison.Ordinal) + && string.Equals(left.Type, right.Type, StringComparison.Ordinal) + && string.Equals(left.Min, right.Min, StringComparison.Ordinal) + && left.MinInclusive == right.MinInclusive + && string.Equals(left.Max, right.Max, StringComparison.Ordinal) + && left.MaxInclusive == right.MaxInclusive + && string.Equals(left.Value, right.Value, StringComparison.Ordinal); +} diff --git a/src/StellaOps.Feedser.Storage.Mongo/Aliases/AliasDocument.cs b/src/StellaOps.Feedser.Storage.Mongo/Aliases/AliasDocument.cs new file mode 100644 index 00000000..b093292e --- /dev/null +++ b/src/StellaOps.Feedser.Storage.Mongo/Aliases/AliasDocument.cs @@ -0,0 +1,38 @@ +using System; +using MongoDB.Bson; +using MongoDB.Bson.Serialization.Attributes; + +namespace StellaOps.Feedser.Storage.Mongo.Aliases; + +[BsonIgnoreExtraElements] +internal sealed class AliasDocument +{ + [BsonId] + public ObjectId Id { get; set; } + + [BsonElement("advisoryKey")] + public string AdvisoryKey { get; set; } = string.Empty; + + [BsonElement("scheme")] + public string Scheme { get; set; } = string.Empty; + + [BsonElement("value")] + public string Value { get; set; } = string.Empty; + + [BsonElement("updatedAt")] + public DateTime UpdatedAt { get; set; } +} + +internal static class AliasDocumentExtensions +{ + public static AliasRecord ToRecord(this AliasDocument document) + { + ArgumentNullException.ThrowIfNull(document); + var updatedAt = DateTime.SpecifyKind(document.UpdatedAt, DateTimeKind.Utc); + return new AliasRecord( + document.AdvisoryKey, + document.Scheme, + document.Value, + new DateTimeOffset(updatedAt)); + } +} diff --git a/src/StellaOps.Feedser.Storage.Mongo/Aliases/AliasStore.cs b/src/StellaOps.Feedser.Storage.Mongo/Aliases/AliasStore.cs new file mode 100644 index 00000000..7b0d71cd --- /dev/null +++ b/src/StellaOps.Feedser.Storage.Mongo/Aliases/AliasStore.cs @@ -0,0 +1,157 @@ +using System.Collections.Generic; +using System.Linq; +using Microsoft.Extensions.Logging; +using MongoDB.Bson; +using MongoDB.Driver; + +namespace StellaOps.Feedser.Storage.Mongo.Aliases; + +public sealed class AliasStore : IAliasStore +{ + private readonly IMongoCollection _collection; + private readonly ILogger _logger; + + public AliasStore(IMongoDatabase database, ILogger logger) + { + _collection = (database ?? throw new ArgumentNullException(nameof(database))) + .GetCollection(MongoStorageDefaults.Collections.Alias); + _logger = logger ?? throw new ArgumentNullException(nameof(logger)); + } + + public async Task ReplaceAsync( + string advisoryKey, + IEnumerable aliases, + DateTimeOffset updatedAt, + CancellationToken cancellationToken) + { + ArgumentException.ThrowIfNullOrWhiteSpace(advisoryKey); + + var aliasList = Normalize(aliases).ToArray(); + var deleteFilter = Builders.Filter.Eq(x => x.AdvisoryKey, advisoryKey); + await _collection.DeleteManyAsync(deleteFilter, cancellationToken).ConfigureAwait(false); + + if (aliasList.Length > 0) + { + var documents = new List(aliasList.Length); + var updatedAtUtc = updatedAt.ToUniversalTime().UtcDateTime; + foreach (var alias in aliasList) + { + documents.Add(new AliasDocument + { + Id = ObjectId.GenerateNewId(), + AdvisoryKey = advisoryKey, + Scheme = alias.Scheme, + Value = alias.Value, + UpdatedAt = updatedAtUtc, + }); + } + + if (documents.Count > 0) + { + await _collection.InsertManyAsync( + documents, + new InsertManyOptions { IsOrdered = false }, + cancellationToken).ConfigureAwait(false); + } + } + + if (aliasList.Length == 0) + { + return new AliasUpsertResult(advisoryKey, Array.Empty()); + } + + var collisions = new List(); + foreach (var alias in aliasList) + { + var filter = Builders.Filter.Eq(x => x.Scheme, alias.Scheme) + & Builders.Filter.Eq(x => x.Value, alias.Value); + + using var cursor = await _collection.FindAsync(filter, cancellationToken: cancellationToken).ConfigureAwait(false); + var advisoryKeys = new HashSet(StringComparer.OrdinalIgnoreCase); + while (await cursor.MoveNextAsync(cancellationToken).ConfigureAwait(false)) + { + foreach (var document in cursor.Current) + { + advisoryKeys.Add(document.AdvisoryKey); + } + } + + if (advisoryKeys.Count <= 1) + { + continue; + } + + var collision = new AliasCollision(alias.Scheme, alias.Value, advisoryKeys.ToArray()); + collisions.Add(collision); + AliasStoreMetrics.RecordCollision(alias.Scheme, advisoryKeys.Count); + _logger.LogWarning( + "Alias collision detected for {Scheme}:{Value}; advisories: {Advisories}", + alias.Scheme, + alias.Value, + string.Join(", ", advisoryKeys)); + } + + return new AliasUpsertResult(advisoryKey, collisions); + } + + public async Task> GetByAliasAsync(string scheme, string value, CancellationToken cancellationToken) + { + ArgumentException.ThrowIfNullOrWhiteSpace(scheme); + ArgumentException.ThrowIfNullOrWhiteSpace(value); + + var normalizedScheme = NormalizeScheme(scheme); + var normalizedValue = value.Trim(); + var filter = Builders.Filter.Eq(x => x.Scheme, normalizedScheme) + & Builders.Filter.Eq(x => x.Value, normalizedValue); + + var documents = await _collection.Find(filter).ToListAsync(cancellationToken).ConfigureAwait(false); + return documents.Select(static d => d.ToRecord()).ToArray(); + } + + public async Task> GetByAdvisoryAsync(string advisoryKey, CancellationToken cancellationToken) + { + ArgumentException.ThrowIfNullOrWhiteSpace(advisoryKey); + var filter = Builders.Filter.Eq(x => x.AdvisoryKey, advisoryKey); + var documents = await _collection.Find(filter).ToListAsync(cancellationToken).ConfigureAwait(false); + return documents.Select(static d => d.ToRecord()).ToArray(); + } + + private static IEnumerable Normalize(IEnumerable aliases) + { + if (aliases is null) + { + yield break; + } + + var seen = new HashSet(StringComparer.Ordinal); + foreach (var alias in aliases) + { + if (alias is null) + { + continue; + } + + var scheme = NormalizeScheme(alias.Scheme); + var value = alias.Value?.Trim(); + if (string.IsNullOrEmpty(value)) + { + continue; + } + + var key = $"{scheme}\u0001{value}"; + if (!seen.Add(key)) + { + continue; + } + + yield return new AliasEntry(scheme, value); + } + } + + private static string NormalizeScheme(string scheme) + { + return string.IsNullOrWhiteSpace(scheme) + ? AliasStoreConstants.UnscopedScheme + : scheme.Trim().ToUpperInvariant(); + } +} diff --git a/src/StellaOps.Feedser.Storage.Mongo/Aliases/AliasStoreConstants.cs b/src/StellaOps.Feedser.Storage.Mongo/Aliases/AliasStoreConstants.cs new file mode 100644 index 00000000..d847acf5 --- /dev/null +++ b/src/StellaOps.Feedser.Storage.Mongo/Aliases/AliasStoreConstants.cs @@ -0,0 +1,7 @@ +namespace StellaOps.Feedser.Storage.Mongo.Aliases; + +public static class AliasStoreConstants +{ + public const string PrimaryScheme = "PRIMARY"; + public const string UnscopedScheme = "UNSCOPED"; +} diff --git a/src/StellaOps.Feedser.Storage.Mongo/Aliases/AliasStoreMetrics.cs b/src/StellaOps.Feedser.Storage.Mongo/Aliases/AliasStoreMetrics.cs new file mode 100644 index 00000000..7117434d --- /dev/null +++ b/src/StellaOps.Feedser.Storage.Mongo/Aliases/AliasStoreMetrics.cs @@ -0,0 +1,22 @@ +using System.Collections.Generic; +using System.Diagnostics.Metrics; + +namespace StellaOps.Feedser.Storage.Mongo.Aliases; + +internal static class AliasStoreMetrics +{ + private static readonly Meter Meter = new("StellaOps.Feedser.Merge"); + + internal static readonly Counter AliasCollisionCounter = Meter.CreateCounter( + "feedser.merge.alias_conflict", + unit: "count", + description: "Number of alias collisions detected when the same alias maps to multiple advisories."); + + public static void RecordCollision(string scheme, int advisoryCount) + { + AliasCollisionCounter.Add( + 1, + new KeyValuePair("scheme", scheme), + new KeyValuePair("advisory_count", advisoryCount)); + } +} diff --git a/src/StellaOps.Feedser.Storage.Mongo/Aliases/IAliasStore.cs b/src/StellaOps.Feedser.Storage.Mongo/Aliases/IAliasStore.cs new file mode 100644 index 00000000..200346a5 --- /dev/null +++ b/src/StellaOps.Feedser.Storage.Mongo/Aliases/IAliasStore.cs @@ -0,0 +1,27 @@ +using System; +using System.Collections.Generic; +using System.Threading; +using System.Threading.Tasks; + +namespace StellaOps.Feedser.Storage.Mongo.Aliases; + +public interface IAliasStore +{ + Task ReplaceAsync( + string advisoryKey, + IEnumerable aliases, + DateTimeOffset updatedAt, + CancellationToken cancellationToken); + + Task> GetByAliasAsync(string scheme, string value, CancellationToken cancellationToken); + + Task> GetByAdvisoryAsync(string advisoryKey, CancellationToken cancellationToken); +} + +public sealed record AliasEntry(string Scheme, string Value); + +public sealed record AliasRecord(string AdvisoryKey, string Scheme, string Value, DateTimeOffset UpdatedAt); + +public sealed record AliasCollision(string Scheme, string Value, IReadOnlyList AdvisoryKeys); + +public sealed record AliasUpsertResult(string AdvisoryKey, IReadOnlyList Collisions); diff --git a/src/StellaOps.Feedser.Storage.Mongo/ChangeHistory/ChangeHistoryDocument.cs b/src/StellaOps.Feedser.Storage.Mongo/ChangeHistory/ChangeHistoryDocument.cs new file mode 100644 index 00000000..d9b5e746 --- /dev/null +++ b/src/StellaOps.Feedser.Storage.Mongo/ChangeHistory/ChangeHistoryDocument.cs @@ -0,0 +1,43 @@ +using System; +using System.Collections.Generic; +using MongoDB.Bson; +using MongoDB.Bson.Serialization.Attributes; + +namespace StellaOps.Feedser.Storage.Mongo.ChangeHistory; + +[BsonIgnoreExtraElements] +public sealed class ChangeHistoryDocument +{ + [BsonId] + public string Id { get; set; } = string.Empty; + + [BsonElement("source")] + public string SourceName { get; set; } = string.Empty; + + [BsonElement("advisoryKey")] + public string AdvisoryKey { get; set; } = string.Empty; + + [BsonElement("documentId")] + public string DocumentId { get; set; } = string.Empty; + + [BsonElement("documentSha256")] + public string DocumentSha256 { get; set; } = string.Empty; + + [BsonElement("currentHash")] + public string CurrentHash { get; set; } = string.Empty; + + [BsonElement("previousHash")] + public string? PreviousHash { get; set; } + + [BsonElement("currentSnapshot")] + public string CurrentSnapshot { get; set; } = string.Empty; + + [BsonElement("previousSnapshot")] + public string? PreviousSnapshot { get; set; } + + [BsonElement("changes")] + public List Changes { get; set; } = new(); + + [BsonElement("capturedAt")] + public DateTime CapturedAt { get; set; } +} diff --git a/src/StellaOps.Feedser.Storage.Mongo/ChangeHistory/ChangeHistoryDocumentExtensions.cs b/src/StellaOps.Feedser.Storage.Mongo/ChangeHistory/ChangeHistoryDocumentExtensions.cs new file mode 100644 index 00000000..a5f7f9ea --- /dev/null +++ b/src/StellaOps.Feedser.Storage.Mongo/ChangeHistory/ChangeHistoryDocumentExtensions.cs @@ -0,0 +1,70 @@ +using System; +using System.Collections.Generic; +using MongoDB.Bson; + +namespace StellaOps.Feedser.Storage.Mongo.ChangeHistory; + +internal static class ChangeHistoryDocumentExtensions +{ + public static ChangeHistoryDocument ToDocument(this ChangeHistoryRecord record) + { + var changes = new List(record.Changes.Count); + foreach (var change in record.Changes) + { + changes.Add(new BsonDocument + { + ["field"] = change.Field, + ["type"] = change.ChangeType, + ["previous"] = change.PreviousValue is null ? BsonNull.Value : new BsonString(change.PreviousValue), + ["current"] = change.CurrentValue is null ? BsonNull.Value : new BsonString(change.CurrentValue), + }); + } + + return new ChangeHistoryDocument + { + Id = record.Id.ToString(), + SourceName = record.SourceName, + AdvisoryKey = record.AdvisoryKey, + DocumentId = record.DocumentId.ToString(), + DocumentSha256 = record.DocumentSha256, + CurrentHash = record.CurrentHash, + PreviousHash = record.PreviousHash, + CurrentSnapshot = record.CurrentSnapshot, + PreviousSnapshot = record.PreviousSnapshot, + Changes = changes, + CapturedAt = record.CapturedAt.UtcDateTime, + }; + } + + public static ChangeHistoryRecord ToRecord(this ChangeHistoryDocument document) + { + var changes = new List(document.Changes.Count); + foreach (var change in document.Changes) + { + var previousValue = change.TryGetValue("previous", out var previousBson) && previousBson is not BsonNull + ? previousBson.AsString + : null; + var currentValue = change.TryGetValue("current", out var currentBson) && currentBson is not BsonNull + ? currentBson.AsString + : null; + var fieldName = change.GetValue("field", "").AsString; + var changeType = change.GetValue("type", "").AsString; + changes.Add(new ChangeHistoryFieldChange(fieldName, changeType, previousValue, currentValue)); + } + + var capturedAtUtc = DateTime.SpecifyKind(document.CapturedAt, DateTimeKind.Utc); + + return new ChangeHistoryRecord( + Guid.Parse(document.Id), + document.SourceName, + document.AdvisoryKey, + Guid.Parse(document.DocumentId), + document.DocumentSha256, + document.CurrentHash, + document.PreviousHash, + document.CurrentSnapshot, + document.PreviousSnapshot, + changes, + new DateTimeOffset(capturedAtUtc)); + } +} diff --git a/src/StellaOps.Feedser.Storage.Mongo/ChangeHistory/ChangeHistoryFieldChange.cs b/src/StellaOps.Feedser.Storage.Mongo/ChangeHistory/ChangeHistoryFieldChange.cs new file mode 100644 index 00000000..c8c80a88 --- /dev/null +++ b/src/StellaOps.Feedser.Storage.Mongo/ChangeHistory/ChangeHistoryFieldChange.cs @@ -0,0 +1,24 @@ +using System; + +namespace StellaOps.Feedser.Storage.Mongo.ChangeHistory; + +public sealed record ChangeHistoryFieldChange +{ + public ChangeHistoryFieldChange(string field, string changeType, string? previousValue, string? currentValue) + { + ArgumentException.ThrowIfNullOrEmpty(field); + ArgumentException.ThrowIfNullOrEmpty(changeType); + Field = field; + ChangeType = changeType; + PreviousValue = previousValue; + CurrentValue = currentValue; + } + + public string Field { get; } + + public string ChangeType { get; } + + public string? PreviousValue { get; } + + public string? CurrentValue { get; } +} diff --git a/src/StellaOps.Feedser.Storage.Mongo/ChangeHistory/ChangeHistoryRecord.cs b/src/StellaOps.Feedser.Storage.Mongo/ChangeHistory/ChangeHistoryRecord.cs new file mode 100644 index 00000000..3b18af98 --- /dev/null +++ b/src/StellaOps.Feedser.Storage.Mongo/ChangeHistory/ChangeHistoryRecord.cs @@ -0,0 +1,62 @@ +using System; +using System.Collections.Generic; + +namespace StellaOps.Feedser.Storage.Mongo.ChangeHistory; + +public sealed class ChangeHistoryRecord +{ + public ChangeHistoryRecord( + Guid id, + string sourceName, + string advisoryKey, + Guid documentId, + string documentSha256, + string currentHash, + string? previousHash, + string currentSnapshot, + string? previousSnapshot, + IReadOnlyList changes, + DateTimeOffset capturedAt) + { + ArgumentException.ThrowIfNullOrEmpty(sourceName); + ArgumentException.ThrowIfNullOrEmpty(advisoryKey); + ArgumentException.ThrowIfNullOrEmpty(documentSha256); + ArgumentException.ThrowIfNullOrEmpty(currentHash); + ArgumentException.ThrowIfNullOrEmpty(currentSnapshot); + ArgumentNullException.ThrowIfNull(changes); + + Id = id; + SourceName = sourceName; + AdvisoryKey = advisoryKey; + DocumentId = documentId; + DocumentSha256 = documentSha256; + CurrentHash = currentHash; + PreviousHash = previousHash; + CurrentSnapshot = currentSnapshot; + PreviousSnapshot = previousSnapshot; + Changes = changes; + CapturedAt = capturedAt; + } + + public Guid Id { get; } + + public string SourceName { get; } + + public string AdvisoryKey { get; } + + public Guid DocumentId { get; } + + public string DocumentSha256 { get; } + + public string CurrentHash { get; } + + public string? PreviousHash { get; } + + public string CurrentSnapshot { get; } + + public string? PreviousSnapshot { get; } + + public IReadOnlyList Changes { get; } + + public DateTimeOffset CapturedAt { get; } +} diff --git a/src/StellaOps.Feedser.Storage.Mongo/ChangeHistory/IChangeHistoryStore.cs b/src/StellaOps.Feedser.Storage.Mongo/ChangeHistory/IChangeHistoryStore.cs new file mode 100644 index 00000000..b409f452 --- /dev/null +++ b/src/StellaOps.Feedser.Storage.Mongo/ChangeHistory/IChangeHistoryStore.cs @@ -0,0 +1,12 @@ +using System.Collections.Generic; +using System.Threading; +using System.Threading.Tasks; + +namespace StellaOps.Feedser.Storage.Mongo.ChangeHistory; + +public interface IChangeHistoryStore +{ + Task AddAsync(ChangeHistoryRecord record, CancellationToken cancellationToken); + + Task> GetRecentAsync(string sourceName, string advisoryKey, int limit, CancellationToken cancellationToken); +} diff --git a/src/StellaOps.Feedser.Storage.Mongo/ChangeHistory/MongoChangeHistoryStore.cs b/src/StellaOps.Feedser.Storage.Mongo/ChangeHistory/MongoChangeHistoryStore.cs new file mode 100644 index 00000000..8f7616ce --- /dev/null +++ b/src/StellaOps.Feedser.Storage.Mongo/ChangeHistory/MongoChangeHistoryStore.cs @@ -0,0 +1,53 @@ +using System; +using System.Collections.Generic; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.Extensions.Logging; +using MongoDB.Driver; + +namespace StellaOps.Feedser.Storage.Mongo.ChangeHistory; + +public sealed class MongoChangeHistoryStore : IChangeHistoryStore +{ + private readonly IMongoCollection _collection; + private readonly ILogger _logger; + + public MongoChangeHistoryStore(IMongoDatabase database, ILogger logger) + { + _collection = (database ?? throw new ArgumentNullException(nameof(database))) + .GetCollection(MongoStorageDefaults.Collections.ChangeHistory); + _logger = logger ?? throw new ArgumentNullException(nameof(logger)); + } + + public async Task AddAsync(ChangeHistoryRecord record, CancellationToken cancellationToken) + { + ArgumentNullException.ThrowIfNull(record); + var document = record.ToDocument(); + await _collection.InsertOneAsync(document, cancellationToken: cancellationToken).ConfigureAwait(false); + _logger.LogDebug("Recorded change history for {Source}/{Advisory} with hash {Hash}", record.SourceName, record.AdvisoryKey, record.CurrentHash); + } + + public async Task> GetRecentAsync(string sourceName, string advisoryKey, int limit, CancellationToken cancellationToken) + { + ArgumentException.ThrowIfNullOrEmpty(sourceName); + ArgumentException.ThrowIfNullOrEmpty(advisoryKey); + if (limit <= 0) + { + limit = 10; + } + + var cursor = await _collection.Find(x => x.SourceName == sourceName && x.AdvisoryKey == advisoryKey) + .SortByDescending(x => x.CapturedAt) + .Limit(limit) + .ToListAsync(cancellationToken) + .ConfigureAwait(false); + + var records = new List(cursor.Count); + foreach (var document in cursor) + { + records.Add(document.ToRecord()); + } + + return records; + } +} diff --git a/src/StellaOps.Feedser.Storage.Mongo/Documents/DocumentDocument.cs b/src/StellaOps.Feedser.Storage.Mongo/Documents/DocumentDocument.cs new file mode 100644 index 00000000..e4af65cb --- /dev/null +++ b/src/StellaOps.Feedser.Storage.Mongo/Documents/DocumentDocument.cs @@ -0,0 +1,131 @@ +using System; +using MongoDB.Bson; +using MongoDB.Bson.Serialization.Attributes; + +namespace StellaOps.Feedser.Storage.Mongo.Documents; + +[BsonIgnoreExtraElements] +public sealed class DocumentDocument +{ + [BsonId] + public string Id { get; set; } = string.Empty; + + [BsonElement("sourceName")] + public string SourceName { get; set; } = string.Empty; + + [BsonElement("uri")] + public string Uri { get; set; } = string.Empty; + + [BsonElement("fetchedAt")] + public DateTime FetchedAt { get; set; } + + [BsonElement("sha256")] + public string Sha256 { get; set; } = string.Empty; + + [BsonElement("status")] + public string Status { get; set; } = string.Empty; + + [BsonElement("contentType")] + [BsonIgnoreIfNull] + public string? ContentType { get; set; } + + [BsonElement("headers")] + [BsonIgnoreIfNull] + public BsonDocument? Headers { get; set; } + + [BsonElement("metadata")] + [BsonIgnoreIfNull] + public BsonDocument? Metadata { get; set; } + + [BsonElement("etag")] + [BsonIgnoreIfNull] + public string? Etag { get; set; } + + [BsonElement("lastModified")] + [BsonIgnoreIfNull] + public DateTime? LastModified { get; set; } + + [BsonElement("expiresAt")] + [BsonIgnoreIfNull] + public DateTime? ExpiresAt { get; set; } + + [BsonElement("gridFsId")] + [BsonIgnoreIfNull] + public ObjectId? GridFsId { get; set; } +} + +internal static class DocumentDocumentExtensions +{ + public static DocumentDocument FromRecord(DocumentRecord record) + { + return new DocumentDocument + { + Id = record.Id.ToString(), + SourceName = record.SourceName, + Uri = record.Uri, + FetchedAt = record.FetchedAt.UtcDateTime, + Sha256 = record.Sha256, + Status = record.Status, + ContentType = record.ContentType, + Headers = ToBson(record.Headers), + Metadata = ToBson(record.Metadata), + Etag = record.Etag, + LastModified = record.LastModified?.UtcDateTime, + GridFsId = record.GridFsId, + ExpiresAt = record.ExpiresAt?.UtcDateTime, + }; + } + + public static DocumentRecord ToRecord(this DocumentDocument document) + { + IReadOnlyDictionary? headers = null; + if (document.Headers is not null) + { + headers = document.Headers.Elements.ToDictionary( + static e => e.Name, + static e => e.Value?.ToString() ?? string.Empty, + StringComparer.Ordinal); + } + + IReadOnlyDictionary? metadata = null; + if (document.Metadata is not null) + { + metadata = document.Metadata.Elements.ToDictionary( + static e => e.Name, + static e => e.Value?.ToString() ?? string.Empty, + StringComparer.Ordinal); + } + + return new DocumentRecord( + Guid.Parse(document.Id), + document.SourceName, + document.Uri, + DateTime.SpecifyKind(document.FetchedAt, DateTimeKind.Utc), + document.Sha256, + document.Status, + document.ContentType, + headers, + metadata, + document.Etag, + document.LastModified.HasValue ? DateTime.SpecifyKind(document.LastModified.Value, DateTimeKind.Utc) : null, + document.GridFsId, + document.ExpiresAt.HasValue ? DateTime.SpecifyKind(document.ExpiresAt.Value, DateTimeKind.Utc) : null); + } + + private static BsonDocument? ToBson(IReadOnlyDictionary? values) + { + if (values is null) + { + return null; + } + + var document = new BsonDocument(); + foreach (var kvp in values) + { + document[kvp.Key] = kvp.Value; + } + + return document; + } + +} diff --git a/src/StellaOps.Feedser.Storage.Mongo/Documents/DocumentRecord.cs b/src/StellaOps.Feedser.Storage.Mongo/Documents/DocumentRecord.cs new file mode 100644 index 00000000..1a371362 --- /dev/null +++ b/src/StellaOps.Feedser.Storage.Mongo/Documents/DocumentRecord.cs @@ -0,0 +1,22 @@ +using MongoDB.Bson; + +namespace StellaOps.Feedser.Storage.Mongo.Documents; + +public sealed record DocumentRecord( + Guid Id, + string SourceName, + string Uri, + DateTimeOffset FetchedAt, + string Sha256, + string Status, + string? ContentType, + IReadOnlyDictionary? Headers, + IReadOnlyDictionary? Metadata, + string? Etag, + DateTimeOffset? LastModified, + ObjectId? GridFsId, + DateTimeOffset? ExpiresAt = null) +{ + public DocumentRecord WithStatus(string status) + => this with { Status = status }; +} diff --git a/src/StellaOps.Feedser.Storage.Mongo/Documents/DocumentStore.cs b/src/StellaOps.Feedser.Storage.Mongo/Documents/DocumentStore.cs new file mode 100644 index 00000000..b6c894c8 --- /dev/null +++ b/src/StellaOps.Feedser.Storage.Mongo/Documents/DocumentStore.cs @@ -0,0 +1,68 @@ +using Microsoft.Extensions.Logging; +using MongoDB.Driver; + +namespace StellaOps.Feedser.Storage.Mongo.Documents; + +public sealed class DocumentStore : IDocumentStore +{ + private readonly IMongoCollection _collection; + private readonly ILogger _logger; + + public DocumentStore(IMongoDatabase database, ILogger logger) + { + _collection = (database ?? throw new ArgumentNullException(nameof(database))) + .GetCollection(MongoStorageDefaults.Collections.Document); + _logger = logger ?? throw new ArgumentNullException(nameof(logger)); + } + + public async Task UpsertAsync(DocumentRecord record, CancellationToken cancellationToken) + { + ArgumentNullException.ThrowIfNull(record); + + var document = DocumentDocumentExtensions.FromRecord(record); + var filter = Builders.Filter.Eq(x => x.SourceName, record.SourceName) + & Builders.Filter.Eq(x => x.Uri, record.Uri); + + var options = new FindOneAndReplaceOptions + { + IsUpsert = true, + ReturnDocument = ReturnDocument.After, + }; + + var replaced = await _collection.FindOneAndReplaceAsync(filter, document, options, cancellationToken).ConfigureAwait(false); + _logger.LogDebug("Upserted document {Source}/{Uri}", record.SourceName, record.Uri); + return (replaced ?? document).ToRecord(); + } + + public async Task FindBySourceAndUriAsync(string sourceName, string uri, CancellationToken cancellationToken) + { + ArgumentException.ThrowIfNullOrEmpty(sourceName); + ArgumentException.ThrowIfNullOrEmpty(uri); + + var filter = Builders.Filter.Eq(x => x.SourceName, sourceName) + & Builders.Filter.Eq(x => x.Uri, uri); + + var document = await _collection.Find(filter).FirstOrDefaultAsync(cancellationToken).ConfigureAwait(false); + return document?.ToRecord(); + } + + public async Task FindAsync(Guid id, CancellationToken cancellationToken) + { + var idValue = id.ToString(); + var document = await _collection.Find(x => x.Id == idValue).FirstOrDefaultAsync(cancellationToken).ConfigureAwait(false); + return document?.ToRecord(); + } + + public async Task UpdateStatusAsync(Guid id, string status, CancellationToken cancellationToken) + { + ArgumentException.ThrowIfNullOrEmpty(status); + + var update = Builders.Update + .Set(x => x.Status, status) + .Set(x => x.LastModified, DateTime.UtcNow); + + var idValue = id.ToString(); + var result = await _collection.UpdateOneAsync(x => x.Id == idValue, update, cancellationToken: cancellationToken).ConfigureAwait(false); + return result.MatchedCount > 0; + } +} diff --git a/src/StellaOps.Feedser.Storage.Mongo/Documents/IDocumentStore.cs b/src/StellaOps.Feedser.Storage.Mongo/Documents/IDocumentStore.cs new file mode 100644 index 00000000..5ce08818 --- /dev/null +++ b/src/StellaOps.Feedser.Storage.Mongo/Documents/IDocumentStore.cs @@ -0,0 +1,12 @@ +namespace StellaOps.Feedser.Storage.Mongo.Documents; + +public interface IDocumentStore +{ + Task UpsertAsync(DocumentRecord record, CancellationToken cancellationToken); + + Task FindBySourceAndUriAsync(string sourceName, string uri, CancellationToken cancellationToken); + + Task FindAsync(Guid id, CancellationToken cancellationToken); + + Task UpdateStatusAsync(Guid id, string status, CancellationToken cancellationToken); +} diff --git a/src/StellaOps.Feedser.Storage.Mongo/Dtos/DtoDocument.cs b/src/StellaOps.Feedser.Storage.Mongo/Dtos/DtoDocument.cs new file mode 100644 index 00000000..a9121c79 --- /dev/null +++ b/src/StellaOps.Feedser.Storage.Mongo/Dtos/DtoDocument.cs @@ -0,0 +1,50 @@ +using System; +using MongoDB.Bson; +using MongoDB.Bson.Serialization.Attributes; + +namespace StellaOps.Feedser.Storage.Mongo.Dtos; + +[BsonIgnoreExtraElements] +public sealed class DtoDocument +{ + [BsonId] + public string Id { get; set; } = string.Empty; + + [BsonElement("documentId")] + public string DocumentId { get; set; } = string.Empty; + + [BsonElement("sourceName")] + public string SourceName { get; set; } = string.Empty; + + [BsonElement("schemaVersion")] + public string SchemaVersion { get; set; } = string.Empty; + + [BsonElement("payload")] + public BsonDocument Payload { get; set; } = new(); + + [BsonElement("validatedAt")] + public DateTime ValidatedAt { get; set; } +} + +internal static class DtoDocumentExtensions +{ + public static DtoDocument FromRecord(DtoRecord record) + => new() + { + Id = record.Id.ToString(), + DocumentId = record.DocumentId.ToString(), + SourceName = record.SourceName, + SchemaVersion = record.SchemaVersion, + Payload = record.Payload ?? new BsonDocument(), + ValidatedAt = record.ValidatedAt.UtcDateTime, + }; + + public static DtoRecord ToRecord(this DtoDocument document) + => new( + Guid.Parse(document.Id), + Guid.Parse(document.DocumentId), + document.SourceName, + document.SchemaVersion, + document.Payload, + DateTime.SpecifyKind(document.ValidatedAt, DateTimeKind.Utc)); +} diff --git a/src/StellaOps.Feedser.Storage.Mongo/Dtos/DtoRecord.cs b/src/StellaOps.Feedser.Storage.Mongo/Dtos/DtoRecord.cs new file mode 100644 index 00000000..21c4eede --- /dev/null +++ b/src/StellaOps.Feedser.Storage.Mongo/Dtos/DtoRecord.cs @@ -0,0 +1,11 @@ +using MongoDB.Bson; + +namespace StellaOps.Feedser.Storage.Mongo.Dtos; + +public sealed record DtoRecord( + Guid Id, + Guid DocumentId, + string SourceName, + string SchemaVersion, + BsonDocument Payload, + DateTimeOffset ValidatedAt); diff --git a/src/StellaOps.Feedser.Storage.Mongo/Dtos/DtoStore.cs b/src/StellaOps.Feedser.Storage.Mongo/Dtos/DtoStore.cs new file mode 100644 index 00000000..8c7c47ea --- /dev/null +++ b/src/StellaOps.Feedser.Storage.Mongo/Dtos/DtoStore.cs @@ -0,0 +1,57 @@ +using Microsoft.Extensions.Logging; +using MongoDB.Driver; + +namespace StellaOps.Feedser.Storage.Mongo.Dtos; + +public sealed class DtoStore : IDtoStore +{ + private readonly IMongoCollection _collection; + private readonly ILogger _logger; + + public DtoStore(IMongoDatabase database, ILogger logger) + { + _collection = (database ?? throw new ArgumentNullException(nameof(database))) + .GetCollection(MongoStorageDefaults.Collections.Dto); + _logger = logger ?? throw new ArgumentNullException(nameof(logger)); + } + + public async Task UpsertAsync(DtoRecord record, CancellationToken cancellationToken) + { + ArgumentNullException.ThrowIfNull(record); + + var document = DtoDocumentExtensions.FromRecord(record); + var documentId = record.DocumentId.ToString(); + var filter = Builders.Filter.Eq(x => x.DocumentId, documentId) + & Builders.Filter.Eq(x => x.SourceName, record.SourceName); + + var options = new FindOneAndReplaceOptions + { + IsUpsert = true, + ReturnDocument = ReturnDocument.After, + }; + + var replaced = await _collection.FindOneAndReplaceAsync(filter, document, options, cancellationToken).ConfigureAwait(false); + _logger.LogDebug("Upserted DTO for {Source}/{DocumentId}", record.SourceName, record.DocumentId); + return (replaced ?? document).ToRecord(); + } + + public async Task FindByDocumentIdAsync(Guid documentId, CancellationToken cancellationToken) + { + var documentIdValue = documentId.ToString(); + var document = await _collection.Find(x => x.DocumentId == documentIdValue) + .FirstOrDefaultAsync(cancellationToken) + .ConfigureAwait(false); + return document?.ToRecord(); + } + + public async Task> GetBySourceAsync(string sourceName, int limit, CancellationToken cancellationToken) + { + var cursor = await _collection.Find(x => x.SourceName == sourceName) + .SortByDescending(x => x.ValidatedAt) + .Limit(limit) + .ToListAsync(cancellationToken) + .ConfigureAwait(false); + + return cursor.Select(static x => x.ToRecord()).ToArray(); + } +} diff --git a/src/StellaOps.Feedser.Storage.Mongo/Dtos/IDtoStore.cs b/src/StellaOps.Feedser.Storage.Mongo/Dtos/IDtoStore.cs new file mode 100644 index 00000000..07806e94 --- /dev/null +++ b/src/StellaOps.Feedser.Storage.Mongo/Dtos/IDtoStore.cs @@ -0,0 +1,10 @@ +namespace StellaOps.Feedser.Storage.Mongo.Dtos; + +public interface IDtoStore +{ + Task UpsertAsync(DtoRecord record, CancellationToken cancellationToken); + + Task FindByDocumentIdAsync(Guid documentId, CancellationToken cancellationToken); + + Task> GetBySourceAsync(string sourceName, int limit, CancellationToken cancellationToken); +} diff --git a/src/StellaOps.Feedser.Storage.Mongo/Exporting/ExportStateDocument.cs b/src/StellaOps.Feedser.Storage.Mongo/Exporting/ExportStateDocument.cs new file mode 100644 index 00000000..a6e87421 --- /dev/null +++ b/src/StellaOps.Feedser.Storage.Mongo/Exporting/ExportStateDocument.cs @@ -0,0 +1,90 @@ +using System.Collections.Generic; +using System.Linq; +using MongoDB.Bson.Serialization.Attributes; + +namespace StellaOps.Feedser.Storage.Mongo.Exporting; + +[BsonIgnoreExtraElements] +public sealed class ExportStateDocument +{ + [BsonId] + public string Id { get; set; } = string.Empty; + + [BsonElement("baseExportId")] + public string? BaseExportId { get; set; } + + [BsonElement("baseDigest")] + public string? BaseDigest { get; set; } + + [BsonElement("lastFullDigest")] + public string? LastFullDigest { get; set; } + + [BsonElement("lastDeltaDigest")] + public string? LastDeltaDigest { get; set; } + + [BsonElement("exportCursor")] + public string? ExportCursor { get; set; } + + [BsonElement("targetRepo")] + public string? TargetRepository { get; set; } + + [BsonElement("exporterVersion")] + public string? ExporterVersion { get; set; } + + [BsonElement("updatedAt")] + public DateTime UpdatedAt { get; set; } + + [BsonElement("files")] + public List? Files { get; set; } +} + +public sealed class ExportStateFileDocument +{ + [BsonElement("path")] + public string Path { get; set; } = string.Empty; + + [BsonElement("length")] + public long Length { get; set; } + + [BsonElement("digest")] + public string Digest { get; set; } = string.Empty; +} + +internal static class ExportStateDocumentExtensions +{ + public static ExportStateDocument FromRecord(ExportStateRecord record) + => new() + { + Id = record.Id, + BaseExportId = record.BaseExportId, + BaseDigest = record.BaseDigest, + LastFullDigest = record.LastFullDigest, + LastDeltaDigest = record.LastDeltaDigest, + ExportCursor = record.ExportCursor, + TargetRepository = record.TargetRepository, + ExporterVersion = record.ExporterVersion, + UpdatedAt = record.UpdatedAt.UtcDateTime, + Files = record.Files.Select(static file => new ExportStateFileDocument + { + Path = file.Path, + Length = file.Length, + Digest = file.Digest, + }).ToList(), + }; + + public static ExportStateRecord ToRecord(this ExportStateDocument document) + => new( + document.Id, + document.BaseExportId, + document.BaseDigest, + document.LastFullDigest, + document.LastDeltaDigest, + document.ExportCursor, + document.TargetRepository, + document.ExporterVersion, + DateTime.SpecifyKind(document.UpdatedAt, DateTimeKind.Utc), + (document.Files ?? new List()) + .Where(static entry => !string.IsNullOrWhiteSpace(entry.Path)) + .Select(static entry => new ExportFileRecord(entry.Path, entry.Length, entry.Digest)) + .ToArray()); +} diff --git a/src/StellaOps.Feedser.Storage.Mongo/Exporting/ExportStateManager.cs b/src/StellaOps.Feedser.Storage.Mongo/Exporting/ExportStateManager.cs new file mode 100644 index 00000000..efd051aa --- /dev/null +++ b/src/StellaOps.Feedser.Storage.Mongo/Exporting/ExportStateManager.cs @@ -0,0 +1,135 @@ +using System; +using System.Collections.Generic; +using System.Threading; +using System.Threading.Tasks; + +namespace StellaOps.Feedser.Storage.Mongo.Exporting; + +/// +/// Helper for exporters to read and persist their export metadata in Mongo-backed storage. +/// +public sealed class ExportStateManager +{ + private readonly IExportStateStore _store; + private readonly TimeProvider _timeProvider; + + public ExportStateManager(IExportStateStore store, TimeProvider? timeProvider = null) + { + _store = store ?? throw new ArgumentNullException(nameof(store)); + _timeProvider = timeProvider ?? TimeProvider.System; + } + + public Task GetAsync(string exporterId, CancellationToken cancellationToken) + { + ArgumentException.ThrowIfNullOrEmpty(exporterId); + return _store.FindAsync(exporterId, cancellationToken); + } + + public async Task StoreFullExportAsync( + string exporterId, + string exportId, + string exportDigest, + string? cursor, + string? targetRepository, + string exporterVersion, + bool resetBaseline, + IReadOnlyList manifest, + CancellationToken cancellationToken) + { + ArgumentException.ThrowIfNullOrEmpty(exporterId); + ArgumentException.ThrowIfNullOrEmpty(exportId); + ArgumentException.ThrowIfNullOrEmpty(exportDigest); + ArgumentException.ThrowIfNullOrEmpty(exporterVersion); + manifest ??= Array.Empty(); + + var existing = await _store.FindAsync(exporterId, cancellationToken).ConfigureAwait(false); + var now = _timeProvider.GetUtcNow(); + + if (existing is null) + { + var resolvedRepository = string.IsNullOrWhiteSpace(targetRepository) ? null : targetRepository; + return await _store.UpsertAsync( + new ExportStateRecord( + exporterId, + BaseExportId: exportId, + BaseDigest: exportDigest, + LastFullDigest: exportDigest, + LastDeltaDigest: null, + ExportCursor: cursor ?? exportDigest, + TargetRepository: resolvedRepository, + ExporterVersion: exporterVersion, + UpdatedAt: now, + Files: manifest), + cancellationToken).ConfigureAwait(false); + } + + var repositorySpecified = !string.IsNullOrWhiteSpace(targetRepository); + var resolvedRepo = repositorySpecified ? targetRepository : existing.TargetRepository; + var repositoryChanged = repositorySpecified + && !string.Equals(existing.TargetRepository, targetRepository, StringComparison.Ordinal); + + var shouldResetBaseline = + resetBaseline + || string.IsNullOrWhiteSpace(existing.BaseExportId) + || string.IsNullOrWhiteSpace(existing.BaseDigest) + || repositoryChanged; + + var updatedRecord = shouldResetBaseline + ? existing with + { + BaseExportId = exportId, + BaseDigest = exportDigest, + LastFullDigest = exportDigest, + LastDeltaDigest = null, + ExportCursor = cursor ?? exportDigest, + TargetRepository = resolvedRepo, + ExporterVersion = exporterVersion, + UpdatedAt = now, + Files = manifest, + } + : existing with + { + LastFullDigest = exportDigest, + LastDeltaDigest = null, + ExportCursor = cursor ?? existing.ExportCursor, + TargetRepository = resolvedRepo, + ExporterVersion = exporterVersion, + UpdatedAt = now, + Files = manifest, + }; + + return await _store.UpsertAsync(updatedRecord, cancellationToken).ConfigureAwait(false); + } + + public async Task StoreDeltaExportAsync( + string exporterId, + string deltaDigest, + string? cursor, + string exporterVersion, + IReadOnlyList manifest, + CancellationToken cancellationToken) + { + ArgumentException.ThrowIfNullOrEmpty(exporterId); + ArgumentException.ThrowIfNullOrEmpty(deltaDigest); + ArgumentException.ThrowIfNullOrEmpty(exporterVersion); + manifest ??= Array.Empty(); + + var existing = await _store.FindAsync(exporterId, cancellationToken).ConfigureAwait(false); + if (existing is null) + { + throw new InvalidOperationException($"Full export state missing for '{exporterId}'."); + } + + var now = _timeProvider.GetUtcNow(); + var record = existing with + { + LastDeltaDigest = deltaDigest, + ExportCursor = cursor ?? existing.ExportCursor, + ExporterVersion = exporterVersion, + UpdatedAt = now, + Files = manifest, + }; + + return await _store.UpsertAsync(record, cancellationToken).ConfigureAwait(false); + } +} diff --git a/src/StellaOps.Feedser.Storage.Mongo/Exporting/ExportStateRecord.cs b/src/StellaOps.Feedser.Storage.Mongo/Exporting/ExportStateRecord.cs new file mode 100644 index 00000000..3d72b44e --- /dev/null +++ b/src/StellaOps.Feedser.Storage.Mongo/Exporting/ExportStateRecord.cs @@ -0,0 +1,15 @@ +namespace StellaOps.Feedser.Storage.Mongo.Exporting; + +public sealed record ExportStateRecord( + string Id, + string? BaseExportId, + string? BaseDigest, + string? LastFullDigest, + string? LastDeltaDigest, + string? ExportCursor, + string? TargetRepository, + string? ExporterVersion, + DateTimeOffset UpdatedAt, + IReadOnlyList Files); + +public sealed record ExportFileRecord(string Path, long Length, string Digest); diff --git a/src/StellaOps.Feedser.Storage.Mongo/Exporting/ExportStateStore.cs b/src/StellaOps.Feedser.Storage.Mongo/Exporting/ExportStateStore.cs new file mode 100644 index 00000000..a45e51a9 --- /dev/null +++ b/src/StellaOps.Feedser.Storage.Mongo/Exporting/ExportStateStore.cs @@ -0,0 +1,43 @@ +using Microsoft.Extensions.Logging; +using MongoDB.Driver; + +namespace StellaOps.Feedser.Storage.Mongo.Exporting; + +public sealed class ExportStateStore : IExportStateStore +{ + private readonly IMongoCollection _collection; + private readonly ILogger _logger; + + public ExportStateStore(IMongoDatabase database, ILogger logger) + { + _collection = (database ?? throw new ArgumentNullException(nameof(database))) + .GetCollection(MongoStorageDefaults.Collections.ExportState); + _logger = logger ?? throw new ArgumentNullException(nameof(logger)); + } + + public async Task UpsertAsync(ExportStateRecord record, CancellationToken cancellationToken) + { + ArgumentNullException.ThrowIfNull(record); + + var document = ExportStateDocumentExtensions.FromRecord(record); + var options = new FindOneAndReplaceOptions + { + IsUpsert = true, + ReturnDocument = ReturnDocument.After, + }; + + var replaced = await _collection.FindOneAndReplaceAsync( + x => x.Id == record.Id, + document, + options, + cancellationToken).ConfigureAwait(false); + _logger.LogDebug("Stored export state {StateId}", record.Id); + return (replaced ?? document).ToRecord(); + } + + public async Task FindAsync(string id, CancellationToken cancellationToken) + { + var document = await _collection.Find(x => x.Id == id).FirstOrDefaultAsync(cancellationToken).ConfigureAwait(false); + return document?.ToRecord(); + } +} diff --git a/src/StellaOps.Feedser.Storage.Mongo/Exporting/IExportStateStore.cs b/src/StellaOps.Feedser.Storage.Mongo/Exporting/IExportStateStore.cs new file mode 100644 index 00000000..3dad3c23 --- /dev/null +++ b/src/StellaOps.Feedser.Storage.Mongo/Exporting/IExportStateStore.cs @@ -0,0 +1,8 @@ +namespace StellaOps.Feedser.Storage.Mongo.Exporting; + +public interface IExportStateStore +{ + Task UpsertAsync(ExportStateRecord record, CancellationToken cancellationToken); + + Task FindAsync(string id, CancellationToken cancellationToken); +} diff --git a/src/StellaOps.Feedser.Storage.Mongo/ISourceStateRepository.cs b/src/StellaOps.Feedser.Storage.Mongo/ISourceStateRepository.cs new file mode 100644 index 00000000..d5e8a98e --- /dev/null +++ b/src/StellaOps.Feedser.Storage.Mongo/ISourceStateRepository.cs @@ -0,0 +1,14 @@ +using MongoDB.Bson; + +namespace StellaOps.Feedser.Storage.Mongo; + +public interface ISourceStateRepository +{ + Task TryGetAsync(string sourceName, CancellationToken cancellationToken); + + Task UpsertAsync(SourceStateRecord record, CancellationToken cancellationToken); + + Task UpdateCursorAsync(string sourceName, BsonDocument cursor, DateTimeOffset completedAt, CancellationToken cancellationToken); + + Task MarkFailureAsync(string sourceName, DateTimeOffset failedAt, TimeSpan? backoff, string? failureReason, CancellationToken cancellationToken); +} diff --git a/src/StellaOps.Feedser.Storage.Mongo/JobLeaseDocument.cs b/src/StellaOps.Feedser.Storage.Mongo/JobLeaseDocument.cs new file mode 100644 index 00000000..ae8b5b34 --- /dev/null +++ b/src/StellaOps.Feedser.Storage.Mongo/JobLeaseDocument.cs @@ -0,0 +1,38 @@ +using MongoDB.Bson.Serialization.Attributes; +using StellaOps.Feedser.Core.Jobs; + +namespace StellaOps.Feedser.Storage.Mongo; + +[BsonIgnoreExtraElements] +public sealed class JobLeaseDocument +{ + [BsonId] + public string Key { get; set; } = string.Empty; + + [BsonElement("holder")] + public string Holder { get; set; } = string.Empty; + + [BsonElement("acquiredAt")] + public DateTime AcquiredAt { get; set; } + + [BsonElement("heartbeatAt")] + public DateTime HeartbeatAt { get; set; } + + [BsonElement("leaseMs")] + public long LeaseMs { get; set; } + + [BsonElement("ttlAt")] + public DateTime TtlAt { get; set; } +} + +internal static class JobLeaseDocumentExtensions +{ + public static JobLease ToLease(this JobLeaseDocument document) + => new( + document.Key, + document.Holder, + DateTime.SpecifyKind(document.AcquiredAt, DateTimeKind.Utc), + DateTime.SpecifyKind(document.HeartbeatAt, DateTimeKind.Utc), + TimeSpan.FromMilliseconds(document.LeaseMs), + DateTime.SpecifyKind(document.TtlAt, DateTimeKind.Utc)); +} diff --git a/src/StellaOps.Feedser.Storage.Mongo/JobRunDocument.cs b/src/StellaOps.Feedser.Storage.Mongo/JobRunDocument.cs new file mode 100644 index 00000000..fabbc3f1 --- /dev/null +++ b/src/StellaOps.Feedser.Storage.Mongo/JobRunDocument.cs @@ -0,0 +1,119 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text.Json; +using MongoDB.Bson; +using MongoDB.Bson.Serialization.Attributes; +using StellaOps.Feedser.Core.Jobs; + +namespace StellaOps.Feedser.Storage.Mongo; + +[BsonIgnoreExtraElements] +public sealed class JobRunDocument +{ + [BsonId] + public string Id { get; set; } = string.Empty; + + [BsonElement("kind")] + public string Kind { get; set; } = string.Empty; + + [BsonElement("status")] + public string Status { get; set; } = JobRunStatus.Pending.ToString(); + + [BsonElement("trigger")] + public string Trigger { get; set; } = string.Empty; + + [BsonElement("parameters")] + public BsonDocument Parameters { get; set; } = new(); + + [BsonElement("parametersHash")] + [BsonIgnoreIfNull] + public string? ParametersHash { get; set; } + + [BsonElement("createdAt")] + public DateTime CreatedAt { get; set; } + + [BsonElement("startedAt")] + [BsonIgnoreIfNull] + public DateTime? StartedAt { get; set; } + + [BsonElement("completedAt")] + [BsonIgnoreIfNull] + public DateTime? CompletedAt { get; set; } + + [BsonElement("error")] + [BsonIgnoreIfNull] + public string? Error { get; set; } + + [BsonElement("timeoutMs")] + [BsonIgnoreIfNull] + public long? TimeoutMs { get; set; } + + [BsonElement("leaseMs")] + [BsonIgnoreIfNull] + public long? LeaseMs { get; set; } +} + +internal static class JobRunDocumentExtensions +{ + public static JobRunDocument FromRequest(JobRunCreateRequest request, Guid id) + { + return new JobRunDocument + { + Id = id.ToString(), + Kind = request.Kind, + Status = JobRunStatus.Pending.ToString(), + Trigger = request.Trigger, + Parameters = request.Parameters is { Count: > 0 } + ? BsonDocument.Parse(JsonSerializer.Serialize(request.Parameters)) + : new BsonDocument(), + ParametersHash = request.ParametersHash, + CreatedAt = request.CreatedAt.UtcDateTime, + TimeoutMs = request.Timeout?.MillisecondsFromTimespan(), + LeaseMs = request.LeaseDuration?.MillisecondsFromTimespan(), + }; + } + + public static JobRunSnapshot ToSnapshot(this JobRunDocument document) + { + var parameters = document.Parameters?.ToDictionary() ?? new Dictionary(); + + return new JobRunSnapshot( + Guid.Parse(document.Id), + document.Kind, + Enum.Parse(document.Status, ignoreCase: true), + DateTime.SpecifyKind(document.CreatedAt, DateTimeKind.Utc), + document.StartedAt.HasValue ? DateTime.SpecifyKind(document.StartedAt.Value, DateTimeKind.Utc) : null, + document.CompletedAt.HasValue ? DateTime.SpecifyKind(document.CompletedAt.Value, DateTimeKind.Utc) : null, + document.Trigger, + document.ParametersHash, + document.Error, + document.TimeoutMs?.MillisecondsToTimespan(), + document.LeaseMs?.MillisecondsToTimespan(), + parameters); + } + + public static Dictionary ToDictionary(this BsonDocument document) + { + return document.Elements.ToDictionary( + static element => element.Name, + static element => element.Value switch + { + BsonString s => (object?)s.AsString, + BsonBoolean b => b.AsBoolean, + BsonInt32 i => i.AsInt32, + BsonInt64 l => l.AsInt64, + BsonDouble d => d.AsDouble, + BsonNull => null, + BsonArray array => array.Select(v => v.IsBsonDocument ? ToDictionary(v.AsBsonDocument) : (object?)v.ToString()).ToArray(), + BsonDocument doc => ToDictionary(doc), + _ => element.Value.ToString(), + }); + } + + private static long MillisecondsFromTimespan(this TimeSpan timeSpan) + => (long)timeSpan.TotalMilliseconds; + + private static TimeSpan MillisecondsToTimespan(this long milliseconds) + => TimeSpan.FromMilliseconds(milliseconds); +} diff --git a/src/StellaOps.Feedser.Storage.Mongo/JpFlags/IJpFlagStore.cs b/src/StellaOps.Feedser.Storage.Mongo/JpFlags/IJpFlagStore.cs new file mode 100644 index 00000000..131ce8b9 --- /dev/null +++ b/src/StellaOps.Feedser.Storage.Mongo/JpFlags/IJpFlagStore.cs @@ -0,0 +1,11 @@ +using System.Threading; +using System.Threading.Tasks; + +namespace StellaOps.Feedser.Storage.Mongo.JpFlags; + +public interface IJpFlagStore +{ + Task UpsertAsync(JpFlagRecord record, CancellationToken cancellationToken); + + Task FindAsync(string advisoryKey, CancellationToken cancellationToken); +} diff --git a/src/StellaOps.Feedser.Storage.Mongo/JpFlags/JpFlagDocument.cs b/src/StellaOps.Feedser.Storage.Mongo/JpFlags/JpFlagDocument.cs new file mode 100644 index 00000000..a493fc51 --- /dev/null +++ b/src/StellaOps.Feedser.Storage.Mongo/JpFlags/JpFlagDocument.cs @@ -0,0 +1,54 @@ +using MongoDB.Bson.Serialization.Attributes; + +namespace StellaOps.Feedser.Storage.Mongo.JpFlags; + +[BsonIgnoreExtraElements] +public sealed class JpFlagDocument +{ + [BsonId] + [BsonElement("advisoryKey")] + public string AdvisoryKey { get; set; } = string.Empty; + + [BsonElement("sourceName")] + public string SourceName { get; set; } = string.Empty; + + [BsonElement("category")] + [BsonIgnoreIfNull] + public string? Category { get; set; } + + [BsonElement("vendorStatus")] + [BsonIgnoreIfNull] + public string? VendorStatus { get; set; } + + [BsonElement("recordedAt")] + public DateTime RecordedAt { get; set; } +} + +internal static class JpFlagDocumentExtensions +{ + public static JpFlagDocument FromRecord(JpFlagRecord record) + { + ArgumentNullException.ThrowIfNull(record); + + return new JpFlagDocument + { + AdvisoryKey = record.AdvisoryKey, + SourceName = record.SourceName, + Category = record.Category, + VendorStatus = record.VendorStatus, + RecordedAt = record.RecordedAt.UtcDateTime, + }; + } + + public static JpFlagRecord ToRecord(this JpFlagDocument document) + { + ArgumentNullException.ThrowIfNull(document); + + return new JpFlagRecord( + document.AdvisoryKey, + document.SourceName, + document.Category, + document.VendorStatus, + DateTime.SpecifyKind(document.RecordedAt, DateTimeKind.Utc)); + } +} diff --git a/src/StellaOps.Feedser.Storage.Mongo/JpFlags/JpFlagRecord.cs b/src/StellaOps.Feedser.Storage.Mongo/JpFlags/JpFlagRecord.cs new file mode 100644 index 00000000..90fa8dc0 --- /dev/null +++ b/src/StellaOps.Feedser.Storage.Mongo/JpFlags/JpFlagRecord.cs @@ -0,0 +1,15 @@ +namespace StellaOps.Feedser.Storage.Mongo.JpFlags; + +/// +/// Captures Japan-specific enrichment flags derived from JVN payloads. +/// +public sealed record JpFlagRecord( + string AdvisoryKey, + string SourceName, + string? Category, + string? VendorStatus, + DateTimeOffset RecordedAt) +{ + public JpFlagRecord WithRecordedAt(DateTimeOffset recordedAt) + => this with { RecordedAt = recordedAt.ToUniversalTime() }; +} diff --git a/src/StellaOps.Feedser.Storage.Mongo/JpFlags/JpFlagStore.cs b/src/StellaOps.Feedser.Storage.Mongo/JpFlags/JpFlagStore.cs new file mode 100644 index 00000000..e5cedaae --- /dev/null +++ b/src/StellaOps.Feedser.Storage.Mongo/JpFlags/JpFlagStore.cs @@ -0,0 +1,39 @@ +using Microsoft.Extensions.Logging; +using MongoDB.Driver; + +namespace StellaOps.Feedser.Storage.Mongo.JpFlags; + +public sealed class JpFlagStore : IJpFlagStore +{ + private readonly IMongoCollection _collection; + private readonly ILogger _logger; + + public JpFlagStore(IMongoDatabase database, ILogger logger) + { + ArgumentNullException.ThrowIfNull(database); + ArgumentNullException.ThrowIfNull(logger); + + _collection = database.GetCollection(MongoStorageDefaults.Collections.JpFlags); + _logger = logger; + } + + public async Task UpsertAsync(JpFlagRecord record, CancellationToken cancellationToken) + { + ArgumentNullException.ThrowIfNull(record); + + var document = JpFlagDocumentExtensions.FromRecord(record); + var filter = Builders.Filter.Eq(x => x.AdvisoryKey, record.AdvisoryKey); + var options = new ReplaceOptions { IsUpsert = true }; + await _collection.ReplaceOneAsync(filter, document, options, cancellationToken).ConfigureAwait(false); + _logger.LogDebug("Upserted jp_flag for {AdvisoryKey}", record.AdvisoryKey); + } + + public async Task FindAsync(string advisoryKey, CancellationToken cancellationToken) + { + ArgumentException.ThrowIfNullOrEmpty(advisoryKey); + + var filter = Builders.Filter.Eq(x => x.AdvisoryKey, advisoryKey); + var document = await _collection.Find(filter).FirstOrDefaultAsync(cancellationToken).ConfigureAwait(false); + return document?.ToRecord(); + } +} diff --git a/src/StellaOps.Feedser.Storage.Mongo/MIGRATIONS.md b/src/StellaOps.Feedser.Storage.Mongo/MIGRATIONS.md new file mode 100644 index 00000000..bca738e1 --- /dev/null +++ b/src/StellaOps.Feedser.Storage.Mongo/MIGRATIONS.md @@ -0,0 +1,37 @@ +# Mongo Schema Migration Playbook + +This module owns the persistent shape of Feedser's MongoDB database. Upgrades must be deterministic and safe to run on live replicas. The `MongoMigrationRunner` executes idempotent migrations on startup immediately after the bootstrapper completes its collection and index checks. + +## Execution Path + +1. `StellaOps.Feedser.WebService` calls `MongoBootstrapper.InitializeAsync()` during startup. +2. Once collections and baseline indexes are ensured, the bootstrapper invokes `MongoMigrationRunner.RunAsync()`. +3. Each `IMongoMigration` implementation is sorted by its `Id` (ordinal compare) and executed exactly once. Completion is recorded in the `schema_migrations` collection. +4. Failures surface during startup and prevent the service from serving traffic, matching our "fail-fast" requirement for storage incompatibilities. + +## Creating a Migration + +1. Implement `IMongoMigration` under `StellaOps.Feedser.Storage.Mongo.Migrations`. Use a monotonically increasing identifier such as `yyyyMMdd_description`. +2. Keep the body idempotent: query state first, drop/re-create indexes only when mismatch is detected, and avoid multi-document transactions unless required. +3. Add the migration to DI in `ServiceCollectionExtensions` so it flows into the runner. +4. Write an integration test that exercises the migration against a Mongo2Go instance to validate behaviour. + +## Current Migrations + +| Id | Description | +| --- | --- | +| `20241005_document_expiry_indexes` | Ensures `document` collection uses the correct TTL/partial index depending on raw document retention settings. | +| `20241005_gridfs_expiry_indexes` | Aligns the GridFS `documents.files` TTL index with retention settings. | + +## Operator Runbook + +- `schema_migrations` records each applied migration (`_id`, `description`, `appliedAt`). Review this collection when auditing upgrades. +- To re-run a migration in a lab, delete the corresponding document from `schema_migrations` and restart the service. **Do not** do this in production unless the migration body is known to be idempotent and safe. +- When changing retention settings (`RawDocumentRetention`), deploy the new configuration and restart Feedser. The migration runner will adjust indexes on the next boot. +- If migrations fail, restart with `Logging__LogLevel__StellaOps.Feedser.Storage.Mongo.Migrations=Debug` to surface diagnostic output. Remediate underlying index/collection drift before retrying. + +## Validating an Upgrade + +1. Run `dotnet test --filter MongoMigrationRunnerTests` to exercise integration coverage. +2. In staging, execute `db.schema_migrations.find().sort({_id:1})` to verify applied migrations and timestamps. +3. Inspect index shapes: `db.document.getIndexes()` and `db.documents.files.getIndexes()` for TTL/partial filter alignment. diff --git a/src/StellaOps.Feedser.Storage.Mongo/MergeEvents/IMergeEventStore.cs b/src/StellaOps.Feedser.Storage.Mongo/MergeEvents/IMergeEventStore.cs new file mode 100644 index 00000000..57df7782 --- /dev/null +++ b/src/StellaOps.Feedser.Storage.Mongo/MergeEvents/IMergeEventStore.cs @@ -0,0 +1,8 @@ +namespace StellaOps.Feedser.Storage.Mongo.MergeEvents; + +public interface IMergeEventStore +{ + Task AppendAsync(MergeEventRecord record, CancellationToken cancellationToken); + + Task> GetRecentAsync(string advisoryKey, int limit, CancellationToken cancellationToken); +} diff --git a/src/StellaOps.Feedser.Storage.Mongo/MergeEvents/MergeEventDocument.cs b/src/StellaOps.Feedser.Storage.Mongo/MergeEvents/MergeEventDocument.cs new file mode 100644 index 00000000..b0db3bb1 --- /dev/null +++ b/src/StellaOps.Feedser.Storage.Mongo/MergeEvents/MergeEventDocument.cs @@ -0,0 +1,101 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using MongoDB.Bson; +using MongoDB.Bson.Serialization.Attributes; + +namespace StellaOps.Feedser.Storage.Mongo.MergeEvents; + +[BsonIgnoreExtraElements] +public sealed class MergeEventDocument +{ + [BsonId] + public string Id { get; set; } = string.Empty; + + [BsonElement("advisoryKey")] + public string AdvisoryKey { get; set; } = string.Empty; + + [BsonElement("beforeHash")] + public byte[] BeforeHash { get; set; } = Array.Empty(); + + [BsonElement("afterHash")] + public byte[] AfterHash { get; set; } = Array.Empty(); + + [BsonElement("mergedAt")] + public DateTime MergedAt { get; set; } + + [BsonElement("inputDocuments")] + public List InputDocuments { get; set; } = new(); + + [BsonElement("fieldDecisions")] + [BsonIgnoreIfNull] + public List? FieldDecisions { get; set; } +} + +internal static class MergeEventDocumentExtensions +{ + public static MergeEventDocument FromRecord(MergeEventRecord record) + => new() + { + Id = record.Id.ToString(), + AdvisoryKey = record.AdvisoryKey, + BeforeHash = record.BeforeHash, + AfterHash = record.AfterHash, + MergedAt = record.MergedAt.UtcDateTime, + InputDocuments = record.InputDocumentIds.Select(static id => id.ToString()).ToList(), + FieldDecisions = record.FieldDecisions.Count == 0 + ? null + : record.FieldDecisions.Select(MergeFieldDecisionDocument.FromRecord).ToList(), + }; + + public static MergeEventRecord ToRecord(this MergeEventDocument document) + => new( + Guid.Parse(document.Id), + document.AdvisoryKey, + document.BeforeHash, + document.AfterHash, + DateTime.SpecifyKind(document.MergedAt, DateTimeKind.Utc), + document.InputDocuments.Select(static value => Guid.Parse(value)).ToList(), + document.FieldDecisions is null + ? Array.Empty() + : document.FieldDecisions.Select(static decision => decision.ToRecord()).ToList()); +} + +[BsonIgnoreExtraElements] +public sealed class MergeFieldDecisionDocument +{ + [BsonElement("field")] + public string Field { get; set; } = string.Empty; + + [BsonElement("selectedSource")] + [BsonIgnoreIfNull] + public string? SelectedSource { get; set; } + + [BsonElement("decisionReason")] + public string DecisionReason { get; set; } = string.Empty; + + [BsonElement("selectedModified")] + [BsonIgnoreIfNull] + public DateTime? SelectedModified { get; set; } + + [BsonElement("consideredSources")] + public List ConsideredSources { get; set; } = new(); + + public static MergeFieldDecisionDocument FromRecord(MergeFieldDecision record) + => new() + { + Field = record.Field, + SelectedSource = record.SelectedSource, + DecisionReason = record.DecisionReason, + SelectedModified = record.SelectedModified?.UtcDateTime, + ConsideredSources = record.ConsideredSources.ToList(), + }; + + public MergeFieldDecision ToRecord() + => new( + Field, + SelectedSource, + DecisionReason, + SelectedModified.HasValue ? DateTime.SpecifyKind(SelectedModified.Value, DateTimeKind.Utc) : null, + ConsideredSources); +} diff --git a/src/StellaOps.Feedser.Storage.Mongo/MergeEvents/MergeEventRecord.cs b/src/StellaOps.Feedser.Storage.Mongo/MergeEvents/MergeEventRecord.cs new file mode 100644 index 00000000..a3e02214 --- /dev/null +++ b/src/StellaOps.Feedser.Storage.Mongo/MergeEvents/MergeEventRecord.cs @@ -0,0 +1,10 @@ +namespace StellaOps.Feedser.Storage.Mongo.MergeEvents; + +public sealed record MergeEventRecord( + Guid Id, + string AdvisoryKey, + byte[] BeforeHash, + byte[] AfterHash, + DateTimeOffset MergedAt, + IReadOnlyList InputDocumentIds, + IReadOnlyList FieldDecisions); diff --git a/src/StellaOps.Feedser.Storage.Mongo/MergeEvents/MergeEventStore.cs b/src/StellaOps.Feedser.Storage.Mongo/MergeEvents/MergeEventStore.cs new file mode 100644 index 00000000..3e77e41c --- /dev/null +++ b/src/StellaOps.Feedser.Storage.Mongo/MergeEvents/MergeEventStore.cs @@ -0,0 +1,36 @@ +using Microsoft.Extensions.Logging; +using MongoDB.Driver; + +namespace StellaOps.Feedser.Storage.Mongo.MergeEvents; + +public sealed class MergeEventStore : IMergeEventStore +{ + private readonly IMongoCollection _collection; + private readonly ILogger _logger; + + public MergeEventStore(IMongoDatabase database, ILogger logger) + { + _collection = (database ?? throw new ArgumentNullException(nameof(database))) + .GetCollection(MongoStorageDefaults.Collections.MergeEvent); + _logger = logger ?? throw new ArgumentNullException(nameof(logger)); + } + + public async Task AppendAsync(MergeEventRecord record, CancellationToken cancellationToken) + { + ArgumentNullException.ThrowIfNull(record); + var document = MergeEventDocumentExtensions.FromRecord(record); + await _collection.InsertOneAsync(document, cancellationToken: cancellationToken).ConfigureAwait(false); + _logger.LogDebug("Appended merge event {MergeId} for {AdvisoryKey}", record.Id, record.AdvisoryKey); + } + + public async Task> GetRecentAsync(string advisoryKey, int limit, CancellationToken cancellationToken) + { + var cursor = await _collection.Find(x => x.AdvisoryKey == advisoryKey) + .SortByDescending(x => x.MergedAt) + .Limit(limit) + .ToListAsync(cancellationToken) + .ConfigureAwait(false); + + return cursor.Select(static x => x.ToRecord()).ToArray(); + } +} diff --git a/src/StellaOps.Feedser.Storage.Mongo/MergeEvents/MergeFieldDecision.cs b/src/StellaOps.Feedser.Storage.Mongo/MergeEvents/MergeFieldDecision.cs new file mode 100644 index 00000000..2d3f753e --- /dev/null +++ b/src/StellaOps.Feedser.Storage.Mongo/MergeEvents/MergeFieldDecision.cs @@ -0,0 +1,8 @@ +namespace StellaOps.Feedser.Storage.Mongo.MergeEvents; + +public sealed record MergeFieldDecision( + string Field, + string? SelectedSource, + string DecisionReason, + DateTimeOffset? SelectedModified, + IReadOnlyList ConsideredSources); diff --git a/src/StellaOps.Feedser.Storage.Mongo/Migrations/EnsureDocumentExpiryIndexesMigration.cs b/src/StellaOps.Feedser.Storage.Mongo/Migrations/EnsureDocumentExpiryIndexesMigration.cs new file mode 100644 index 00000000..12732a81 --- /dev/null +++ b/src/StellaOps.Feedser.Storage.Mongo/Migrations/EnsureDocumentExpiryIndexesMigration.cs @@ -0,0 +1,146 @@ +using System; +using System.Linq; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.Extensions.Options; +using MongoDB.Bson; +using MongoDB.Driver; + +namespace StellaOps.Feedser.Storage.Mongo.Migrations; + +internal sealed class EnsureDocumentExpiryIndexesMigration : IMongoMigration +{ + private readonly MongoStorageOptions _options; + + public EnsureDocumentExpiryIndexesMigration(IOptions options) + { + ArgumentNullException.ThrowIfNull(options); + _options = options.Value; + } + + public string Id => "20241005_document_expiry_indexes"; + + public string Description => "Ensure document.expiresAt index matches configured retention"; + + public async Task ApplyAsync(IMongoDatabase database, CancellationToken cancellationToken) + { + ArgumentNullException.ThrowIfNull(database); + + var needsTtl = _options.RawDocumentRetention > TimeSpan.Zero; + var collection = database.GetCollection(MongoStorageDefaults.Collections.Document); + + using var cursor = await collection.Indexes.ListAsync(cancellationToken).ConfigureAwait(false); + var indexes = await cursor.ToListAsync(cancellationToken).ConfigureAwait(false); + + var ttlIndex = indexes.FirstOrDefault(x => TryGetName(x, out var name) && string.Equals(name, "document_expiresAt_ttl", StringComparison.Ordinal)); + var nonTtlIndex = indexes.FirstOrDefault(x => TryGetName(x, out var name) && string.Equals(name, "document_expiresAt", StringComparison.Ordinal)); + + if (needsTtl) + { + var shouldRebuild = ttlIndex is null || !IndexMatchesTtlExpectations(ttlIndex); + if (shouldRebuild) + { + if (ttlIndex is not null) + { + await collection.Indexes.DropOneAsync("document_expiresAt_ttl", cancellationToken).ConfigureAwait(false); + } + + if (nonTtlIndex is not null) + { + await collection.Indexes.DropOneAsync("document_expiresAt", cancellationToken).ConfigureAwait(false); + } + + var options = new CreateIndexOptions + { + Name = "document_expiresAt_ttl", + ExpireAfter = TimeSpan.Zero, + PartialFilterExpression = Builders.Filter.Exists("expiresAt", true), + }; + + var keys = Builders.IndexKeys.Ascending("expiresAt"); + await collection.Indexes.CreateOneAsync(new CreateIndexModel(keys, options), cancellationToken: cancellationToken).ConfigureAwait(false); + } + else if (nonTtlIndex is not null) + { + await collection.Indexes.DropOneAsync("document_expiresAt", cancellationToken).ConfigureAwait(false); + } + } + else + { + if (ttlIndex is not null) + { + await collection.Indexes.DropOneAsync("document_expiresAt_ttl", cancellationToken).ConfigureAwait(false); + } + + var shouldRebuild = nonTtlIndex is null || !IndexMatchesNonTtlExpectations(nonTtlIndex); + if (shouldRebuild) + { + if (nonTtlIndex is not null) + { + await collection.Indexes.DropOneAsync("document_expiresAt", cancellationToken).ConfigureAwait(false); + } + + var options = new CreateIndexOptions + { + Name = "document_expiresAt", + PartialFilterExpression = Builders.Filter.Exists("expiresAt", true), + }; + + var keys = Builders.IndexKeys.Ascending("expiresAt"); + await collection.Indexes.CreateOneAsync(new CreateIndexModel(keys, options), cancellationToken: cancellationToken).ConfigureAwait(false); + } + } + } + + private static bool IndexMatchesTtlExpectations(BsonDocument index) + { + if (!index.TryGetValue("expireAfterSeconds", out var expireAfter) || expireAfter.ToDouble() != 0) + { + return false; + } + + if (!index.TryGetValue("partialFilterExpression", out var partialFilter) || partialFilter is not BsonDocument partialDoc) + { + return false; + } + + if (!partialDoc.TryGetValue("expiresAt", out var expiresAtRule) || expiresAtRule is not BsonDocument expiresAtDoc) + { + return false; + } + + return expiresAtDoc.Contains("$exists") && expiresAtDoc["$exists"].ToBoolean(); + } + + private static bool IndexMatchesNonTtlExpectations(BsonDocument index) + { + if (index.Contains("expireAfterSeconds")) + { + return false; + } + + if (!index.TryGetValue("partialFilterExpression", out var partialFilter) || partialFilter is not BsonDocument partialDoc) + { + return false; + } + + if (!partialDoc.TryGetValue("expiresAt", out var expiresAtRule) || expiresAtRule is not BsonDocument expiresAtDoc) + { + return false; + } + + return expiresAtDoc.Contains("$exists") && expiresAtDoc["$exists"].ToBoolean(); + } + + private static bool TryGetName(BsonDocument index, out string name) + { + if (index.TryGetValue("name", out var value) && value.IsString) + { + name = value.AsString; + return true; + } + + name = string.Empty; + return false; + } +} diff --git a/src/StellaOps.Feedser.Storage.Mongo/Migrations/EnsureGridFsExpiryIndexesMigration.cs b/src/StellaOps.Feedser.Storage.Mongo/Migrations/EnsureGridFsExpiryIndexesMigration.cs new file mode 100644 index 00000000..158f7aa7 --- /dev/null +++ b/src/StellaOps.Feedser.Storage.Mongo/Migrations/EnsureGridFsExpiryIndexesMigration.cs @@ -0,0 +1,95 @@ +using System; +using System.Linq; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.Extensions.Options; +using MongoDB.Bson; +using MongoDB.Driver; + +namespace StellaOps.Feedser.Storage.Mongo.Migrations; + +internal sealed class EnsureGridFsExpiryIndexesMigration : IMongoMigration +{ + private readonly MongoStorageOptions _options; + + public EnsureGridFsExpiryIndexesMigration(IOptions options) + { + ArgumentNullException.ThrowIfNull(options); + _options = options.Value; + } + + public string Id => "20241005_gridfs_expiry_indexes"; + + public string Description => "Ensure GridFS metadata.expiresAt TTL index reflects retention settings"; + + public async Task ApplyAsync(IMongoDatabase database, CancellationToken cancellationToken) + { + ArgumentNullException.ThrowIfNull(database); + + var needsTtl = _options.RawDocumentRetention > TimeSpan.Zero; + var collection = database.GetCollection("documents.files"); + + using var cursor = await collection.Indexes.ListAsync(cancellationToken).ConfigureAwait(false); + var indexes = await cursor.ToListAsync(cancellationToken).ConfigureAwait(false); + + var ttlIndex = indexes.FirstOrDefault(x => TryGetName(x, out var name) && string.Equals(name, "gridfs_files_expiresAt_ttl", StringComparison.Ordinal)); + + if (needsTtl) + { + var shouldRebuild = ttlIndex is null || !IndexMatchesTtlExpectations(ttlIndex); + if (shouldRebuild) + { + if (ttlIndex is not null) + { + await collection.Indexes.DropOneAsync("gridfs_files_expiresAt_ttl", cancellationToken).ConfigureAwait(false); + } + + var keys = Builders.IndexKeys.Ascending("metadata.expiresAt"); + var options = new CreateIndexOptions + { + Name = "gridfs_files_expiresAt_ttl", + ExpireAfter = TimeSpan.Zero, + PartialFilterExpression = Builders.Filter.Exists("metadata.expiresAt", true), + }; + + await collection.Indexes.CreateOneAsync(new CreateIndexModel(keys, options), cancellationToken: cancellationToken).ConfigureAwait(false); + } + } + else if (ttlIndex is not null) + { + await collection.Indexes.DropOneAsync("gridfs_files_expiresAt_ttl", cancellationToken).ConfigureAwait(false); + } + } + + private static bool IndexMatchesTtlExpectations(BsonDocument index) + { + if (!index.TryGetValue("expireAfterSeconds", out var expireAfter) || expireAfter.ToDouble() != 0) + { + return false; + } + + if (!index.TryGetValue("partialFilterExpression", out var partialFilter) || partialFilter is not BsonDocument partialDoc) + { + return false; + } + + if (!partialDoc.TryGetValue("metadata.expiresAt", out var expiresAtRule) || expiresAtRule is not BsonDocument expiresAtDoc) + { + return false; + } + + return expiresAtDoc.Contains("$exists") && expiresAtDoc["$exists"].ToBoolean(); + } + + private static bool TryGetName(BsonDocument index, out string name) + { + if (index.TryGetValue("name", out var value) && value.IsString) + { + name = value.AsString; + return true; + } + + name = string.Empty; + return false; + } +} diff --git a/src/StellaOps.Feedser.Storage.Mongo/Migrations/IMongoMigration.cs b/src/StellaOps.Feedser.Storage.Mongo/Migrations/IMongoMigration.cs new file mode 100644 index 00000000..0a0b845c --- /dev/null +++ b/src/StellaOps.Feedser.Storage.Mongo/Migrations/IMongoMigration.cs @@ -0,0 +1,24 @@ +using MongoDB.Driver; + +namespace StellaOps.Feedser.Storage.Mongo.Migrations; + +/// +/// Represents a single, idempotent MongoDB migration. +/// +public interface IMongoMigration +{ + /// + /// Unique identifier for the migration. Sorting is performed using ordinal comparison. + /// + string Id { get; } + + /// + /// Short description surfaced in logs to aid runbooks. + /// + string Description { get; } + + /// + /// Executes the migration. + /// + Task ApplyAsync(IMongoDatabase database, CancellationToken cancellationToken); +} diff --git a/src/StellaOps.Feedser.Storage.Mongo/Migrations/MongoMigrationDocument.cs b/src/StellaOps.Feedser.Storage.Mongo/Migrations/MongoMigrationDocument.cs new file mode 100644 index 00000000..268996e6 --- /dev/null +++ b/src/StellaOps.Feedser.Storage.Mongo/Migrations/MongoMigrationDocument.cs @@ -0,0 +1,18 @@ +using MongoDB.Bson; +using MongoDB.Bson.Serialization.Attributes; + +namespace StellaOps.Feedser.Storage.Mongo.Migrations; + +[BsonIgnoreExtraElements] +internal sealed class MongoMigrationDocument +{ + [BsonId] + public string Id { get; set; } = string.Empty; + + [BsonElement("description")] + [BsonIgnoreIfNull] + public string? Description { get; set; } + + [BsonElement("appliedAt")] + public DateTime AppliedAtUtc { get; set; } +} diff --git a/src/StellaOps.Feedser.Storage.Mongo/Migrations/MongoMigrationRunner.cs b/src/StellaOps.Feedser.Storage.Mongo/Migrations/MongoMigrationRunner.cs new file mode 100644 index 00000000..0256a448 --- /dev/null +++ b/src/StellaOps.Feedser.Storage.Mongo/Migrations/MongoMigrationRunner.cs @@ -0,0 +1,102 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.Extensions.Logging; +using MongoDB.Driver; + +namespace StellaOps.Feedser.Storage.Mongo.Migrations; + +/// +/// Executes pending schema migrations tracked inside MongoDB to keep upgrades deterministic. +/// +public sealed class MongoMigrationRunner +{ + private readonly IMongoDatabase _database; + private readonly IReadOnlyList _migrations; + private readonly ILogger _logger; + private readonly TimeProvider _timeProvider; + + public MongoMigrationRunner( + IMongoDatabase database, + IEnumerable migrations, + ILogger logger, + TimeProvider? timeProvider = null) + { + _database = database ?? throw new ArgumentNullException(nameof(database)); + _logger = logger ?? throw new ArgumentNullException(nameof(logger)); + _timeProvider = timeProvider ?? TimeProvider.System; + _migrations = (migrations ?? throw new ArgumentNullException(nameof(migrations))) + .OrderBy(m => m.Id, StringComparer.Ordinal) + .ToArray(); + } + + public async Task RunAsync(CancellationToken cancellationToken) + { + if (_migrations.Count == 0) + { + return; + } + + var collection = _database.GetCollection(MongoStorageDefaults.Collections.Migrations); + await EnsureCollectionExistsAsync(_database, cancellationToken).ConfigureAwait(false); + + var appliedIds = await LoadAppliedMigrationIdsAsync(collection, cancellationToken).ConfigureAwait(false); + foreach (var migration in _migrations) + { + if (appliedIds.Contains(migration.Id, StringComparer.Ordinal)) + { + continue; + } + + _logger.LogInformation("Applying Mongo migration {MigrationId}: {Description}", migration.Id, migration.Description); + try + { + await migration.ApplyAsync(_database, cancellationToken).ConfigureAwait(false); + var document = new MongoMigrationDocument + { + Id = migration.Id, + Description = string.IsNullOrWhiteSpace(migration.Description) ? null : migration.Description, + AppliedAtUtc = _timeProvider.GetUtcNow().UtcDateTime, + }; + + await collection.InsertOneAsync(document, cancellationToken: cancellationToken).ConfigureAwait(false); + _logger.LogInformation("Mongo migration {MigrationId} applied", migration.Id); + } + catch (Exception ex) + { + _logger.LogError(ex, "Mongo migration {MigrationId} failed", migration.Id); + throw; + } + } + } + + private static async Task> LoadAppliedMigrationIdsAsync( + IMongoCollection collection, + CancellationToken cancellationToken) + { + using var cursor = await collection.FindAsync(FilterDefinition.Empty, cancellationToken: cancellationToken).ConfigureAwait(false); + var applied = await cursor.ToListAsync(cancellationToken).ConfigureAwait(false); + var set = new HashSet(StringComparer.Ordinal); + foreach (var document in applied) + { + if (!string.IsNullOrWhiteSpace(document.Id)) + { + set.Add(document.Id); + } + } + + return set; + } + + private static async Task EnsureCollectionExistsAsync(IMongoDatabase database, CancellationToken cancellationToken) + { + using var cursor = await database.ListCollectionNamesAsync(cancellationToken: cancellationToken).ConfigureAwait(false); + var names = await cursor.ToListAsync(cancellationToken).ConfigureAwait(false); + if (!names.Contains(MongoStorageDefaults.Collections.Migrations, StringComparer.Ordinal)) + { + await database.CreateCollectionAsync(MongoStorageDefaults.Collections.Migrations, cancellationToken: cancellationToken).ConfigureAwait(false); + } + } +} diff --git a/src/StellaOps.Feedser.Storage.Mongo/Migrations/SemVerStyleBackfillMigration.cs b/src/StellaOps.Feedser.Storage.Mongo/Migrations/SemVerStyleBackfillMigration.cs new file mode 100644 index 00000000..d54dc669 --- /dev/null +++ b/src/StellaOps.Feedser.Storage.Mongo/Migrations/SemVerStyleBackfillMigration.cs @@ -0,0 +1,81 @@ +using System; +using System.Collections.Generic; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Options; +using MongoDB.Bson; +using MongoDB.Driver; +using StellaOps.Feedser.Models; +using StellaOps.Feedser.Storage.Mongo.Advisories; + +namespace StellaOps.Feedser.Storage.Mongo.Migrations; + +public sealed class SemVerStyleBackfillMigration : IMongoMigration +{ + private readonly MongoStorageOptions _options; + private readonly ILogger _logger; + + public SemVerStyleBackfillMigration(IOptions options, ILogger logger) + { + _options = options?.Value ?? throw new ArgumentNullException(nameof(options)); + _logger = logger ?? throw new ArgumentNullException(nameof(logger)); + } + + public string Id => "20251011-semver-style-backfill"; + + public string Description => "Populate advisory.normalizedVersions for existing documents when SemVer style storage is enabled."; + + public async Task ApplyAsync(IMongoDatabase database, CancellationToken cancellationToken) + { + if (!_options.EnableSemVerStyle) + { + _logger.LogInformation("SemVer style flag disabled; skipping migration {MigrationId}.", Id); + return; + } + + var collection = database.GetCollection(MongoStorageDefaults.Collections.Advisory); + var filter = Builders.Filter.Or( + Builders.Filter.Exists(doc => doc.NormalizedVersions, false), + Builders.Filter.Where(doc => doc.NormalizedVersions == null || doc.NormalizedVersions.Count == 0)); + + var batchSize = Math.Max(25, _options.BackfillBatchSize); + while (true) + { + var pending = await collection.Find(filter) + .SortBy(doc => doc.AdvisoryKey) + .Limit(batchSize) + .ToListAsync(cancellationToken) + .ConfigureAwait(false); + + if (pending.Count == 0) + { + break; + } + + var updates = new List>(pending.Count); + foreach (var document in pending) + { + var advisory = CanonicalJsonSerializer.Deserialize(document.Payload.ToJson()); + var normalized = NormalizedVersionDocumentFactory.Create(advisory); + + if (normalized is null || normalized.Count == 0) + { + updates.Add(new UpdateOneModel( + Builders.Filter.Eq(doc => doc.AdvisoryKey, document.AdvisoryKey), + Builders.Update.Unset(doc => doc.NormalizedVersions))); + continue; + } + + updates.Add(new UpdateOneModel( + Builders.Filter.Eq(doc => doc.AdvisoryKey, document.AdvisoryKey), + Builders.Update.Set(doc => doc.NormalizedVersions, normalized))); + } + + if (updates.Count > 0) + { + await collection.BulkWriteAsync(updates, cancellationToken: cancellationToken).ConfigureAwait(false); + } + } + } +} diff --git a/src/StellaOps.Feedser.Storage.Mongo/MongoBootstrapper.cs b/src/StellaOps.Feedser.Storage.Mongo/MongoBootstrapper.cs new file mode 100644 index 00000000..48c04d85 --- /dev/null +++ b/src/StellaOps.Feedser.Storage.Mongo/MongoBootstrapper.cs @@ -0,0 +1,308 @@ +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Options; +using MongoDB.Bson; +using MongoDB.Driver; +using StellaOps.Feedser.Storage.Mongo.Migrations; + +namespace StellaOps.Feedser.Storage.Mongo; + +/// +/// Ensures required collections and indexes exist before the service begins processing. +/// +public sealed class MongoBootstrapper +{ + private const string RawDocumentBucketName = "documents"; + private static readonly string[] RequiredCollections = + { + MongoStorageDefaults.Collections.Source, + MongoStorageDefaults.Collections.SourceState, + MongoStorageDefaults.Collections.Document, + MongoStorageDefaults.Collections.Dto, + MongoStorageDefaults.Collections.Advisory, + MongoStorageDefaults.Collections.Alias, + MongoStorageDefaults.Collections.Affected, + MongoStorageDefaults.Collections.Reference, + MongoStorageDefaults.Collections.KevFlag, + MongoStorageDefaults.Collections.RuFlags, + MongoStorageDefaults.Collections.JpFlags, + MongoStorageDefaults.Collections.PsirtFlags, + MongoStorageDefaults.Collections.MergeEvent, + MongoStorageDefaults.Collections.ExportState, + MongoStorageDefaults.Collections.ChangeHistory, + MongoStorageDefaults.Collections.Locks, + MongoStorageDefaults.Collections.Jobs, + MongoStorageDefaults.Collections.Migrations, + }; + + private readonly IMongoDatabase _database; + private readonly MongoStorageOptions _options; + private readonly ILogger _logger; + private readonly MongoMigrationRunner _migrationRunner; + + public MongoBootstrapper( + IMongoDatabase database, + IOptions options, + ILogger logger, + MongoMigrationRunner migrationRunner) + { + _database = database ?? throw new ArgumentNullException(nameof(database)); + _options = options?.Value ?? throw new ArgumentNullException(nameof(options)); + _logger = logger ?? throw new ArgumentNullException(nameof(logger)); + _migrationRunner = migrationRunner ?? throw new ArgumentNullException(nameof(migrationRunner)); + } + + public async Task InitializeAsync(CancellationToken cancellationToken) + { + var existingCollections = await ListCollectionsAsync(cancellationToken).ConfigureAwait(false); + + foreach (var collectionName in RequiredCollections) + { + if (!existingCollections.Contains(collectionName)) + { + await _database.CreateCollectionAsync(collectionName, cancellationToken: cancellationToken).ConfigureAwait(false); + _logger.LogInformation("Created Mongo collection {Collection}", collectionName); + } + } + + await Task.WhenAll( + EnsureLocksIndexesAsync(cancellationToken), + EnsureJobsIndexesAsync(cancellationToken), + EnsureAdvisoryIndexesAsync(cancellationToken), + EnsureDocumentsIndexesAsync(cancellationToken), + EnsureDtoIndexesAsync(cancellationToken), + EnsureAliasIndexesAsync(cancellationToken), + EnsureAffectedIndexesAsync(cancellationToken), + EnsureReferenceIndexesAsync(cancellationToken), + EnsureSourceStateIndexesAsync(cancellationToken), + EnsurePsirtFlagIndexesAsync(cancellationToken), + EnsureChangeHistoryIndexesAsync(cancellationToken), + EnsureGridFsIndexesAsync(cancellationToken)).ConfigureAwait(false); + + await _migrationRunner.RunAsync(cancellationToken).ConfigureAwait(false); + + _logger.LogInformation("Mongo bootstrapper completed"); + } + + private async Task> ListCollectionsAsync(CancellationToken cancellationToken) + { + using var cursor = await _database.ListCollectionNamesAsync(cancellationToken: cancellationToken).ConfigureAwait(false); + var list = await cursor.ToListAsync(cancellationToken).ConfigureAwait(false); + return new HashSet(list, StringComparer.Ordinal); + } + + private Task EnsureLocksIndexesAsync(CancellationToken cancellationToken) + { + var collection = _database.GetCollection(MongoStorageDefaults.Collections.Locks); + var indexes = new List> + { + new( + Builders.IndexKeys.Ascending("ttlAt"), + new CreateIndexOptions { Name = "ttl_at_ttl", ExpireAfter = TimeSpan.Zero }), + }; + + return collection.Indexes.CreateManyAsync(indexes, cancellationToken); + } + + private Task EnsureJobsIndexesAsync(CancellationToken cancellationToken) + { + var collection = _database.GetCollection(MongoStorageDefaults.Collections.Jobs); + var indexes = new List> + { + new( + Builders.IndexKeys.Descending("createdAt"), + new CreateIndexOptions { Name = "jobs_createdAt_desc" }), + new( + Builders.IndexKeys.Ascending("kind").Descending("createdAt"), + new CreateIndexOptions { Name = "jobs_kind_createdAt" }), + new( + Builders.IndexKeys.Ascending("status").Descending("createdAt"), + new CreateIndexOptions { Name = "jobs_status_createdAt" }), + }; + + return collection.Indexes.CreateManyAsync(indexes, cancellationToken); + } + + private Task EnsureAdvisoryIndexesAsync(CancellationToken cancellationToken) + { + var collection = _database.GetCollection(MongoStorageDefaults.Collections.Advisory); + var indexes = new List> + { + new( + Builders.IndexKeys.Ascending("advisoryKey"), + new CreateIndexOptions { Name = "advisory_key_unique", Unique = true }), + new( + Builders.IndexKeys.Descending("modified"), + new CreateIndexOptions { Name = "advisory_modified_desc" }), + new( + Builders.IndexKeys.Descending("published"), + new CreateIndexOptions { Name = "advisory_published_desc" }), + }; + + return collection.Indexes.CreateManyAsync(indexes, cancellationToken); + } + + private Task EnsureDocumentsIndexesAsync(CancellationToken cancellationToken) + { + var collection = _database.GetCollection(MongoStorageDefaults.Collections.Document); + var indexes = new List> + { + new( + Builders.IndexKeys.Ascending("sourceName").Ascending("uri"), + new CreateIndexOptions { Name = "document_source_uri_unique", Unique = true }), + new( + Builders.IndexKeys.Descending("fetchedAt"), + new CreateIndexOptions { Name = "document_fetchedAt_desc" }), + }; + + var expiresKey = Builders.IndexKeys.Ascending("expiresAt"); + var expiresOptions = new CreateIndexOptions + { + Name = _options.RawDocumentRetention > TimeSpan.Zero ? "document_expiresAt_ttl" : "document_expiresAt", + PartialFilterExpression = Builders.Filter.Exists("expiresAt", true), + }; + + if (_options.RawDocumentRetention > TimeSpan.Zero) + { + expiresOptions.ExpireAfter = TimeSpan.Zero; + } + + indexes.Add(new CreateIndexModel(expiresKey, expiresOptions)); + + return collection.Indexes.CreateManyAsync(indexes, cancellationToken); + } + + private Task EnsureAliasIndexesAsync(CancellationToken cancellationToken) + { + var collection = _database.GetCollection(MongoStorageDefaults.Collections.Alias); + var indexes = new List> + { + new( + Builders.IndexKeys.Ascending("scheme").Ascending("value"), + new CreateIndexOptions { Name = "alias_scheme_value", Unique = false }), + }; + + return collection.Indexes.CreateManyAsync(indexes, cancellationToken); + } + + private Task EnsureGridFsIndexesAsync(CancellationToken cancellationToken) + { + if (_options.RawDocumentRetention <= TimeSpan.Zero) + { + return Task.CompletedTask; + } + + var collectionName = $"{RawDocumentBucketName}.files"; + var collection = _database.GetCollection(collectionName); + var indexes = new List> + { + new( + Builders.IndexKeys.Ascending("metadata.expiresAt"), + new CreateIndexOptions + { + Name = "gridfs_files_expiresAt_ttl", + ExpireAfter = TimeSpan.Zero, + PartialFilterExpression = Builders.Filter.Exists("metadata.expiresAt", true), + }), + }; + + return collection.Indexes.CreateManyAsync(indexes, cancellationToken); + } + + private Task EnsureAffectedIndexesAsync(CancellationToken cancellationToken) + { + var collection = _database.GetCollection(MongoStorageDefaults.Collections.Affected); + var indexes = new List> + { + new( + Builders.IndexKeys.Ascending("platform").Ascending("name"), + new CreateIndexOptions { Name = "affected_platform_name" }), + new( + Builders.IndexKeys.Ascending("advisoryId"), + new CreateIndexOptions { Name = "affected_advisoryId" }), + }; + + return collection.Indexes.CreateManyAsync(indexes, cancellationToken); + } + + private Task EnsureReferenceIndexesAsync(CancellationToken cancellationToken) + { + var collection = _database.GetCollection(MongoStorageDefaults.Collections.Reference); + var indexes = new List> + { + new( + Builders.IndexKeys.Ascending("url"), + new CreateIndexOptions { Name = "reference_url" }), + new( + Builders.IndexKeys.Ascending("advisoryId"), + new CreateIndexOptions { Name = "reference_advisoryId" }), + }; + + return collection.Indexes.CreateManyAsync(indexes, cancellationToken); + } + + private Task EnsureSourceStateIndexesAsync(CancellationToken cancellationToken) + { + var collection = _database.GetCollection(MongoStorageDefaults.Collections.SourceState); + var indexes = new List> + { + new( + Builders.IndexKeys.Ascending("sourceName"), + new CreateIndexOptions { Name = "source_state_unique", Unique = true }), + }; + + return collection.Indexes.CreateManyAsync(indexes, cancellationToken); + } + + private Task EnsureDtoIndexesAsync(CancellationToken cancellationToken) + { + var collection = _database.GetCollection(MongoStorageDefaults.Collections.Dto); + var indexes = new List> + { + new( + Builders.IndexKeys.Ascending("documentId"), + new CreateIndexOptions { Name = "dto_documentId" }), + new( + Builders.IndexKeys.Ascending("sourceName").Descending("validatedAt"), + new CreateIndexOptions { Name = "dto_source_validated" }), + }; + + return collection.Indexes.CreateManyAsync(indexes, cancellationToken); + } + + private async Task EnsurePsirtFlagIndexesAsync(CancellationToken cancellationToken) + { + var collection = _database.GetCollection(MongoStorageDefaults.Collections.PsirtFlags); + try + { + await collection.Indexes.DropOneAsync("psirt_advisoryKey_unique", cancellationToken).ConfigureAwait(false); + } + catch (MongoCommandException ex) when (ex.CodeName == "IndexNotFound") + { + } + + var index = new CreateIndexModel( + Builders.IndexKeys.Ascending("vendor"), + new CreateIndexOptions { Name = "psirt_vendor" }); + + await collection.Indexes.CreateOneAsync(index, cancellationToken: cancellationToken).ConfigureAwait(false); + } + + private Task EnsureChangeHistoryIndexesAsync(CancellationToken cancellationToken) + { + var collection = _database.GetCollection(MongoStorageDefaults.Collections.ChangeHistory); + var indexes = new List> + { + new( + Builders.IndexKeys.Ascending("source").Ascending("advisoryKey").Descending("capturedAt"), + new CreateIndexOptions { Name = "history_source_advisory_capturedAt" }), + new( + Builders.IndexKeys.Descending("capturedAt"), + new CreateIndexOptions { Name = "history_capturedAt" }), + new( + Builders.IndexKeys.Ascending("documentId"), + new CreateIndexOptions { Name = "history_documentId" }) + }; + + return collection.Indexes.CreateManyAsync(indexes, cancellationToken); + } +} diff --git a/src/StellaOps.Feedser.Storage.Mongo/MongoJobStore.cs b/src/StellaOps.Feedser.Storage.Mongo/MongoJobStore.cs new file mode 100644 index 00000000..ce0a3fab --- /dev/null +++ b/src/StellaOps.Feedser.Storage.Mongo/MongoJobStore.cs @@ -0,0 +1,194 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using Microsoft.Extensions.Logging; +using MongoDB.Bson; +using MongoDB.Bson.Serialization; +using MongoDB.Driver; +using StellaOps.Feedser.Core.Jobs; + +namespace StellaOps.Feedser.Storage.Mongo; + +public sealed class MongoJobStore : IJobStore +{ + private static readonly string PendingStatus = JobRunStatus.Pending.ToString(); + private static readonly string RunningStatus = JobRunStatus.Running.ToString(); + + private readonly IMongoCollection _collection; + private readonly ILogger _logger; + + public MongoJobStore(IMongoCollection collection, ILogger logger) + { + _collection = collection ?? throw new ArgumentNullException(nameof(collection)); + _logger = logger ?? throw new ArgumentNullException(nameof(logger)); + } + + public async Task CreateAsync(JobRunCreateRequest request, CancellationToken cancellationToken) + { + var runId = Guid.NewGuid(); + var document = JobRunDocumentExtensions.FromRequest(request, runId); + + await _collection.InsertOneAsync(document, cancellationToken: cancellationToken).ConfigureAwait(false); + _logger.LogDebug("Created job run {RunId} for {Kind} with trigger {Trigger}", runId, request.Kind, request.Trigger); + + return document.ToSnapshot(); + } + + public async Task TryStartAsync(Guid runId, DateTimeOffset startedAt, CancellationToken cancellationToken) + { + var runIdValue = runId.ToString(); + var filter = Builders.Filter.Eq(x => x.Id, runIdValue) + & Builders.Filter.Eq(x => x.Status, PendingStatus); + + var update = Builders.Update + .Set(x => x.Status, RunningStatus) + .Set(x => x.StartedAt, startedAt.UtcDateTime); + + var result = await _collection.FindOneAndUpdateAsync( + filter, + update, + new FindOneAndUpdateOptions + { + ReturnDocument = ReturnDocument.After, + }, + cancellationToken).ConfigureAwait(false); + + if (result is null) + { + _logger.LogDebug("Failed to start job run {RunId}; status transition rejected", runId); + return null; + } + + return result.ToSnapshot(); + } + + public async Task TryCompleteAsync(Guid runId, JobRunCompletion completion, CancellationToken cancellationToken) + { + var runIdValue = runId.ToString(); + var filter = Builders.Filter.Eq(x => x.Id, runIdValue) + & Builders.Filter.In(x => x.Status, new[] { PendingStatus, RunningStatus }); + + var update = Builders.Update + .Set(x => x.Status, completion.Status.ToString()) + .Set(x => x.CompletedAt, completion.CompletedAt.UtcDateTime) + .Set(x => x.Error, completion.Error); + + var result = await _collection.FindOneAndUpdateAsync( + filter, + update, + new FindOneAndUpdateOptions + { + ReturnDocument = ReturnDocument.After, + }, + cancellationToken).ConfigureAwait(false); + + if (result is null) + { + _logger.LogWarning("Failed to mark job run {RunId} as {Status}", runId, completion.Status); + return null; + } + + return result.ToSnapshot(); + } + + public async Task FindAsync(Guid runId, CancellationToken cancellationToken) + { + var cursor = await _collection.FindAsync(x => x.Id == runId.ToString(), cancellationToken: cancellationToken).ConfigureAwait(false); + var document = await cursor.FirstOrDefaultAsync(cancellationToken).ConfigureAwait(false); + return document?.ToSnapshot(); + } + + public async Task> GetRecentRunsAsync(string? kind, int limit, CancellationToken cancellationToken) + { + if (limit <= 0) + { + return Array.Empty(); + } + + var filter = string.IsNullOrWhiteSpace(kind) + ? Builders.Filter.Empty + : Builders.Filter.Eq(x => x.Kind, kind); + + var cursor = await _collection.Find(filter) + .SortByDescending(x => x.CreatedAt) + .Limit(limit) + .ToListAsync(cancellationToken) + .ConfigureAwait(false); + + return cursor.Select(static doc => doc.ToSnapshot()).ToArray(); + } + + public async Task> GetActiveRunsAsync(CancellationToken cancellationToken) + { + var filter = Builders.Filter.In(x => x.Status, new[] { PendingStatus, RunningStatus }); + var cursor = await _collection.Find(filter) + .SortByDescending(x => x.CreatedAt) + .ToListAsync(cancellationToken) + .ConfigureAwait(false); + + return cursor.Select(static doc => doc.ToSnapshot()).ToArray(); + } + + public async Task GetLastRunAsync(string kind, CancellationToken cancellationToken) + { + var cursor = await _collection.Find(x => x.Kind == kind) + .SortByDescending(x => x.CreatedAt) + .Limit(1) + .ToListAsync(cancellationToken) + .ConfigureAwait(false); + + return cursor.FirstOrDefault()?.ToSnapshot(); + } + + public async Task> GetLastRunsAsync(IEnumerable kinds, CancellationToken cancellationToken) + { + if (kinds is null) + { + throw new ArgumentNullException(nameof(kinds)); + } + + var kindList = kinds + .Where(static kind => !string.IsNullOrWhiteSpace(kind)) + .Select(static kind => kind.Trim()) + .Distinct(StringComparer.Ordinal) + .ToArray(); + + if (kindList.Length == 0) + { + return new Dictionary(StringComparer.Ordinal); + } + + var matchStage = new BsonDocument("$match", new BsonDocument("kind", new BsonDocument("$in", new BsonArray(kindList)))); + var sortStage = new BsonDocument("$sort", new BsonDocument("createdAt", -1)); + var groupStage = new BsonDocument("$group", new BsonDocument + { + { "_id", "$kind" }, + { "document", new BsonDocument("$first", "$$ROOT") } + }); + + var pipeline = new[] { matchStage, sortStage, groupStage }; + + var aggregate = await _collection.Aggregate(pipeline) + .ToListAsync(cancellationToken) + .ConfigureAwait(false); + + var results = new Dictionary(StringComparer.Ordinal); + foreach (var element in aggregate) + { + if (!element.TryGetValue("_id", out var idValue) || idValue.BsonType != BsonType.String) + { + continue; + } + + if (!element.TryGetValue("document", out var documentValue) || documentValue.BsonType != BsonType.Document) + { + continue; + } + + var document = BsonSerializer.Deserialize(documentValue.AsBsonDocument); + results[idValue.AsString] = document.ToSnapshot(); + } + + return results; + } +} diff --git a/src/StellaOps.Feedser.Storage.Mongo/MongoLeaseStore.cs b/src/StellaOps.Feedser.Storage.Mongo/MongoLeaseStore.cs new file mode 100644 index 00000000..5df66eca --- /dev/null +++ b/src/StellaOps.Feedser.Storage.Mongo/MongoLeaseStore.cs @@ -0,0 +1,116 @@ +using Microsoft.Extensions.Logging; +using MongoDB.Driver; +using StellaOps.Feedser.Core.Jobs; + +namespace StellaOps.Feedser.Storage.Mongo; + +public sealed class MongoLeaseStore : ILeaseStore +{ + private readonly IMongoCollection _collection; + private readonly ILogger _logger; + + public MongoLeaseStore(IMongoCollection collection, ILogger logger) + { + _collection = collection ?? throw new ArgumentNullException(nameof(collection)); + _logger = logger ?? throw new ArgumentNullException(nameof(logger)); + } + + public async Task TryAcquireAsync(string key, string holder, TimeSpan leaseDuration, DateTimeOffset now, CancellationToken cancellationToken) + { + var nowUtc = now.UtcDateTime; + var ttlUtc = nowUtc.Add(leaseDuration); + + var filter = Builders.Filter.Eq(x => x.Key, key) + & Builders.Filter.Or( + Builders.Filter.Lte(x => x.TtlAt, nowUtc), + Builders.Filter.Eq(x => x.Holder, holder)); + + var update = Builders.Update + .Set(x => x.Holder, holder) + .Set(x => x.AcquiredAt, nowUtc) + .Set(x => x.HeartbeatAt, nowUtc) + .Set(x => x.LeaseMs, (long)leaseDuration.TotalMilliseconds) + .Set(x => x.TtlAt, ttlUtc); + + var options = new FindOneAndUpdateOptions + { + ReturnDocument = ReturnDocument.After, + }; + + var updated = await _collection.FindOneAndUpdateAsync(filter, update, options, cancellationToken).ConfigureAwait(false); + if (updated is not null) + { + _logger.LogDebug("Lease {Key} acquired by {Holder}", key, holder); + return updated.ToLease(); + } + + try + { + var document = new JobLeaseDocument + { + Key = key, + Holder = holder, + AcquiredAt = nowUtc, + HeartbeatAt = nowUtc, + LeaseMs = (long)leaseDuration.TotalMilliseconds, + TtlAt = ttlUtc, + }; + + await _collection.InsertOneAsync(document, cancellationToken: cancellationToken).ConfigureAwait(false); + _logger.LogDebug("Lease {Key} inserted for {Holder}", key, holder); + return document.ToLease(); + } + catch (MongoWriteException ex) when (ex.WriteError.Category == ServerErrorCategory.DuplicateKey) + { + _logger.LogDebug(ex, "Lease {Key} already held by another process", key); + return null; + } + } + + public async Task HeartbeatAsync(string key, string holder, TimeSpan leaseDuration, DateTimeOffset now, CancellationToken cancellationToken) + { + var nowUtc = now.UtcDateTime; + var ttlUtc = nowUtc.Add(leaseDuration); + + var filter = Builders.Filter.Eq(x => x.Key, key) + & Builders.Filter.Eq(x => x.Holder, holder); + + var update = Builders.Update + .Set(x => x.HeartbeatAt, nowUtc) + .Set(x => x.LeaseMs, (long)leaseDuration.TotalMilliseconds) + .Set(x => x.TtlAt, ttlUtc); + + var updated = await _collection.FindOneAndUpdateAsync( + filter, + update, + new FindOneAndUpdateOptions + { + ReturnDocument = ReturnDocument.After, + }, + cancellationToken).ConfigureAwait(false); + + if (updated is null) + { + _logger.LogDebug("Heartbeat rejected for lease {Key} held by {Holder}", key, holder); + } + + return updated?.ToLease(); + } + + public async Task ReleaseAsync(string key, string holder, CancellationToken cancellationToken) + { + var result = await _collection.DeleteOneAsync( + Builders.Filter.Eq(x => x.Key, key) + & Builders.Filter.Eq(x => x.Holder, holder), + cancellationToken).ConfigureAwait(false); + + if (result.DeletedCount == 0) + { + _logger.LogDebug("Lease {Key} not released by {Holder}; no matching document", key, holder); + return false; + } + + _logger.LogDebug("Lease {Key} released by {Holder}", key, holder); + return true; + } +} diff --git a/src/StellaOps.Feedser.Storage.Mongo/MongoSourceStateRepository.cs b/src/StellaOps.Feedser.Storage.Mongo/MongoSourceStateRepository.cs new file mode 100644 index 00000000..f3a88687 --- /dev/null +++ b/src/StellaOps.Feedser.Storage.Mongo/MongoSourceStateRepository.cs @@ -0,0 +1,112 @@ +using Microsoft.Extensions.Logging; +using MongoDB.Bson; +using MongoDB.Driver; + +namespace StellaOps.Feedser.Storage.Mongo; + +public sealed class MongoSourceStateRepository : ISourceStateRepository +{ + private readonly IMongoCollection _collection; + private const int MaxFailureReasonLength = 1024; + + private readonly ILogger _logger; + + public MongoSourceStateRepository(IMongoDatabase database, ILogger logger) + { + _collection = (database ?? throw new ArgumentNullException(nameof(database))) + .GetCollection(MongoStorageDefaults.Collections.SourceState); + _logger = logger ?? throw new ArgumentNullException(nameof(logger)); + } + + public async Task TryGetAsync(string sourceName, CancellationToken cancellationToken) + { + var cursor = await _collection.FindAsync(x => x.SourceName == sourceName, cancellationToken: cancellationToken).ConfigureAwait(false); + var document = await cursor.FirstOrDefaultAsync(cancellationToken).ConfigureAwait(false); + return document?.ToRecord(); + } + + public async Task UpsertAsync(SourceStateRecord record, CancellationToken cancellationToken) + { + var document = SourceStateDocumentExtensions.FromRecord(record with { UpdatedAt = DateTimeOffset.UtcNow }); + await _collection.ReplaceOneAsync( + x => x.SourceName == record.SourceName, + document, + new ReplaceOptions { IsUpsert = true }, + cancellationToken).ConfigureAwait(false); + + _logger.LogDebug("Upserted source state for {Source}", record.SourceName); + return document.ToRecord(); + } + + public async Task UpdateCursorAsync(string sourceName, BsonDocument cursor, DateTimeOffset completedAt, CancellationToken cancellationToken) + { + ArgumentException.ThrowIfNullOrEmpty(sourceName); + var update = Builders.Update + .Set(x => x.Cursor, cursor ?? new BsonDocument()) + .Set(x => x.LastSuccess, completedAt.UtcDateTime) + .Set(x => x.FailCount, 0) + .Set(x => x.BackoffUntil, (DateTime?)null) + .Set(x => x.LastFailureReason, null) + .Set(x => x.UpdatedAt, DateTime.UtcNow) + .SetOnInsert(x => x.SourceName, sourceName); + + var options = new FindOneAndUpdateOptions + { + ReturnDocument = ReturnDocument.After, + IsUpsert = true, + }; + + var document = await _collection + .FindOneAndUpdateAsync( + x => x.SourceName == sourceName, + update, + options, + cancellationToken) + .ConfigureAwait(false); + return document?.ToRecord(); + } + + public async Task MarkFailureAsync(string sourceName, DateTimeOffset failedAt, TimeSpan? backoff, string? failureReason, CancellationToken cancellationToken) + { + ArgumentException.ThrowIfNullOrEmpty(sourceName); + var reasonValue = NormalizeFailureReason(failureReason); + var update = Builders.Update + .Inc(x => x.FailCount, 1) + .Set(x => x.LastFailure, failedAt.UtcDateTime) + .Set(x => x.BackoffUntil, backoff.HasValue ? failedAt.UtcDateTime.Add(backoff.Value) : null) + .Set(x => x.LastFailureReason, reasonValue) + .Set(x => x.UpdatedAt, DateTime.UtcNow) + .SetOnInsert(x => x.SourceName, sourceName); + + var options = new FindOneAndUpdateOptions + { + ReturnDocument = ReturnDocument.After, + IsUpsert = true, + }; + + var document = await _collection + .FindOneAndUpdateAsync( + x => x.SourceName == sourceName, + update, + options, + cancellationToken) + .ConfigureAwait(false); + return document?.ToRecord(); + } + + private static string? NormalizeFailureReason(string? reason) + { + if (string.IsNullOrWhiteSpace(reason)) + { + return null; + } + + var trimmed = reason.Trim(); + if (trimmed.Length <= MaxFailureReasonLength) + { + return trimmed; + } + + return trimmed[..MaxFailureReasonLength]; + } +} diff --git a/src/StellaOps.Feedser.Storage.Mongo/MongoStorageDefaults.cs b/src/StellaOps.Feedser.Storage.Mongo/MongoStorageDefaults.cs new file mode 100644 index 00000000..a3c07576 --- /dev/null +++ b/src/StellaOps.Feedser.Storage.Mongo/MongoStorageDefaults.cs @@ -0,0 +1,28 @@ +namespace StellaOps.Feedser.Storage.Mongo; + +public static class MongoStorageDefaults +{ + public const string DefaultDatabaseName = "feedser"; + + public static class Collections + { + public const string Source = "source"; + public const string SourceState = "source_state"; + public const string Document = "document"; + public const string Dto = "dto"; + public const string Advisory = "advisory"; + public const string Alias = "alias"; + public const string Affected = "affected"; + public const string Reference = "reference"; + public const string KevFlag = "kev_flag"; + public const string RuFlags = "ru_flags"; + public const string JpFlags = "jp_flags"; + public const string PsirtFlags = "psirt_flags"; + public const string MergeEvent = "merge_event"; + public const string ExportState = "export_state"; + public const string Locks = "locks"; + public const string Jobs = "jobs"; + public const string Migrations = "schema_migrations"; + public const string ChangeHistory = "source_change_history"; + } +} diff --git a/src/StellaOps.Feedser.Storage.Mongo/MongoStorageOptions.cs b/src/StellaOps.Feedser.Storage.Mongo/MongoStorageOptions.cs new file mode 100644 index 00000000..fe7315b7 --- /dev/null +++ b/src/StellaOps.Feedser.Storage.Mongo/MongoStorageOptions.cs @@ -0,0 +1,93 @@ +using MongoDB.Driver; + +namespace StellaOps.Feedser.Storage.Mongo; + +public sealed class MongoStorageOptions +{ + public string ConnectionString { get; set; } = string.Empty; + + public string? DatabaseName { get; set; } + + public TimeSpan CommandTimeout { get; set; } = TimeSpan.FromSeconds(30); + + /// + /// Retention period for raw documents (document + DTO + GridFS payloads). + /// Set to to disable automatic expiry. + /// + public TimeSpan RawDocumentRetention { get; set; } = TimeSpan.FromDays(45); + + /// + /// Additional grace period applied on top of before TTL purges old rows. + /// Allows the retention background service to delete GridFS blobs first. + /// + public TimeSpan RawDocumentRetentionTtlGrace { get; set; } = TimeSpan.FromDays(1); + + /// + /// Interval between retention sweeps. Only used when is greater than zero. + /// + public TimeSpan RawDocumentRetentionSweepInterval { get; set; } = TimeSpan.FromHours(6); + + /// + /// Enables dual-write of normalized SemVer analytics for affected packages. + /// + public bool EnableSemVerStyle { get; set; } = true; + + /// + /// Batch size used by backfill migrations when repopulating normalized version documents. + /// + public int BackfillBatchSize { get; set; } = 250; + + public string GetDatabaseName() + { + if (!string.IsNullOrWhiteSpace(DatabaseName)) + { + return DatabaseName.Trim(); + } + + if (!string.IsNullOrWhiteSpace(ConnectionString)) + { + var url = MongoUrl.Create(ConnectionString); + if (!string.IsNullOrWhiteSpace(url.DatabaseName)) + { + return url.DatabaseName; + } + } + + return MongoStorageDefaults.DefaultDatabaseName; + } + + public void EnsureValid() + { + if (string.IsNullOrWhiteSpace(ConnectionString)) + { + throw new InvalidOperationException("Mongo connection string is not configured."); + } + + if (CommandTimeout <= TimeSpan.Zero) + { + throw new InvalidOperationException("Command timeout must be greater than zero."); + } + + if (RawDocumentRetention < TimeSpan.Zero) + { + throw new InvalidOperationException("Raw document retention cannot be negative."); + } + + if (RawDocumentRetentionTtlGrace < TimeSpan.Zero) + { + throw new InvalidOperationException("Raw document retention TTL grace cannot be negative."); + } + + if (RawDocumentRetention > TimeSpan.Zero && RawDocumentRetentionSweepInterval <= TimeSpan.Zero) + { + throw new InvalidOperationException("Raw document retention sweep interval must be positive when retention is enabled."); + } + + if (BackfillBatchSize <= 0) + { + throw new InvalidOperationException("Backfill batch size must be greater than zero."); + } + + _ = GetDatabaseName(); + } +} diff --git a/src/StellaOps.Feedser.Storage.Mongo/Properties/AssemblyInfo.cs b/src/StellaOps.Feedser.Storage.Mongo/Properties/AssemblyInfo.cs new file mode 100644 index 00000000..6a4ba72a --- /dev/null +++ b/src/StellaOps.Feedser.Storage.Mongo/Properties/AssemblyInfo.cs @@ -0,0 +1,3 @@ +using System.Runtime.CompilerServices; + +[assembly: InternalsVisibleTo("StellaOps.Feedser.Storage.Mongo.Tests")] diff --git a/src/StellaOps.Feedser.Storage.Mongo/PsirtFlags/IPsirtFlagStore.cs b/src/StellaOps.Feedser.Storage.Mongo/PsirtFlags/IPsirtFlagStore.cs new file mode 100644 index 00000000..568f64d3 --- /dev/null +++ b/src/StellaOps.Feedser.Storage.Mongo/PsirtFlags/IPsirtFlagStore.cs @@ -0,0 +1,11 @@ +using System.Threading; +using System.Threading.Tasks; + +namespace StellaOps.Feedser.Storage.Mongo.PsirtFlags; + +public interface IPsirtFlagStore +{ + Task UpsertAsync(PsirtFlagRecord record, CancellationToken cancellationToken); + + Task FindAsync(string advisoryKey, CancellationToken cancellationToken); +} diff --git a/src/StellaOps.Feedser.Storage.Mongo/PsirtFlags/PsirtFlagDocument.cs b/src/StellaOps.Feedser.Storage.Mongo/PsirtFlags/PsirtFlagDocument.cs new file mode 100644 index 00000000..e572588e --- /dev/null +++ b/src/StellaOps.Feedser.Storage.Mongo/PsirtFlags/PsirtFlagDocument.cs @@ -0,0 +1,52 @@ +using MongoDB.Bson.Serialization.Attributes; + +namespace StellaOps.Feedser.Storage.Mongo.PsirtFlags; + +[BsonIgnoreExtraElements] +public sealed class PsirtFlagDocument +{ + [BsonId] + [BsonElement("advisoryKey")] + public string AdvisoryKey { get; set; } = string.Empty; + + [BsonElement("vendor")] + public string Vendor { get; set; } = string.Empty; + + [BsonElement("sourceName")] + public string SourceName { get; set; } = string.Empty; + + [BsonElement("advisoryIdText")] + public string AdvisoryIdText { get; set; } = string.Empty; + + [BsonElement("flaggedAt")] + public DateTime FlaggedAt { get; set; } +} + +internal static class PsirtFlagDocumentExtensions +{ + public static PsirtFlagDocument FromRecord(PsirtFlagRecord record) + { + ArgumentNullException.ThrowIfNull(record); + + return new PsirtFlagDocument + { + AdvisoryKey = string.IsNullOrWhiteSpace(record.AdvisoryKey) ? record.AdvisoryIdText : record.AdvisoryKey, + Vendor = record.Vendor, + SourceName = record.SourceName, + AdvisoryIdText = record.AdvisoryIdText, + FlaggedAt = record.FlaggedAt.UtcDateTime, + }; + } + + public static PsirtFlagRecord ToRecord(this PsirtFlagDocument document) + { + ArgumentNullException.ThrowIfNull(document); + + return new PsirtFlagRecord( + document.AdvisoryKey, + document.Vendor, + document.SourceName, + document.AdvisoryIdText, + DateTime.SpecifyKind(document.FlaggedAt, DateTimeKind.Utc)); + } +} diff --git a/src/StellaOps.Feedser.Storage.Mongo/PsirtFlags/PsirtFlagRecord.cs b/src/StellaOps.Feedser.Storage.Mongo/PsirtFlags/PsirtFlagRecord.cs new file mode 100644 index 00000000..c3216ff5 --- /dev/null +++ b/src/StellaOps.Feedser.Storage.Mongo/PsirtFlags/PsirtFlagRecord.cs @@ -0,0 +1,15 @@ +namespace StellaOps.Feedser.Storage.Mongo.PsirtFlags; + +/// +/// Describes a PSIRT precedence flag for a canonical advisory. +/// +public sealed record PsirtFlagRecord( + string AdvisoryKey, + string Vendor, + string SourceName, + string AdvisoryIdText, + DateTimeOffset FlaggedAt) +{ + public PsirtFlagRecord WithFlaggedAt(DateTimeOffset flaggedAt) + => this with { FlaggedAt = flaggedAt.ToUniversalTime() }; +} diff --git a/src/StellaOps.Feedser.Storage.Mongo/PsirtFlags/PsirtFlagStore.cs b/src/StellaOps.Feedser.Storage.Mongo/PsirtFlags/PsirtFlagStore.cs new file mode 100644 index 00000000..4f42e232 --- /dev/null +++ b/src/StellaOps.Feedser.Storage.Mongo/PsirtFlags/PsirtFlagStore.cs @@ -0,0 +1,50 @@ +using System.Threading; +using System.Threading.Tasks; +using Microsoft.Extensions.Logging; +using MongoDB.Driver; + +namespace StellaOps.Feedser.Storage.Mongo.PsirtFlags; + +public sealed class PsirtFlagStore : IPsirtFlagStore +{ + private readonly IMongoCollection _collection; + private readonly ILogger _logger; + + public PsirtFlagStore(IMongoDatabase database, ILogger logger) + { + ArgumentNullException.ThrowIfNull(database); + ArgumentNullException.ThrowIfNull(logger); + + _collection = database.GetCollection(MongoStorageDefaults.Collections.PsirtFlags); + _logger = logger; + } + + public async Task UpsertAsync(PsirtFlagRecord record, CancellationToken cancellationToken) + { + ArgumentNullException.ThrowIfNull(record); + ArgumentException.ThrowIfNullOrEmpty(record.AdvisoryKey); + + var document = PsirtFlagDocumentExtensions.FromRecord(record); + var filter = Builders.Filter.Eq(x => x.AdvisoryKey, record.AdvisoryKey); + var options = new ReplaceOptions { IsUpsert = true }; + + try + { + await _collection.ReplaceOneAsync(filter, document, options, cancellationToken).ConfigureAwait(false); + _logger.LogDebug("Upserted PSIRT flag for {AdvisoryKey}", record.AdvisoryKey); + } + catch (MongoWriteException ex) when (ex.WriteError?.Category == ServerErrorCategory.DuplicateKey) + { + _logger.LogWarning(ex, "Duplicate PSIRT flag detected for {AdvisoryKey}", record.AdvisoryKey); + } + } + + public async Task FindAsync(string advisoryKey, CancellationToken cancellationToken) + { + ArgumentException.ThrowIfNullOrEmpty(advisoryKey); + + var filter = Builders.Filter.Eq(x => x.AdvisoryKey, advisoryKey); + var document = await _collection.Find(filter).FirstOrDefaultAsync(cancellationToken).ConfigureAwait(false); + return document?.ToRecord(); + } +} diff --git a/src/StellaOps.Feedser.Storage.Mongo/RawDocumentRetentionService.cs b/src/StellaOps.Feedser.Storage.Mongo/RawDocumentRetentionService.cs new file mode 100644 index 00000000..60027b57 --- /dev/null +++ b/src/StellaOps.Feedser.Storage.Mongo/RawDocumentRetentionService.cs @@ -0,0 +1,155 @@ +using Microsoft.Extensions.Hosting; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Options; +using MongoDB.Driver; +using MongoDB.Driver.GridFS; +using StellaOps.Feedser.Storage.Mongo.Documents; +using StellaOps.Feedser.Storage.Mongo.Dtos; + +namespace StellaOps.Feedser.Storage.Mongo; + +/// +/// Periodically purges expired raw documents, associated DTO payloads, and GridFS content. +/// Complements TTL indexes by ensuring deterministic cleanup before Mongo's background sweeper runs. +/// +internal sealed class RawDocumentRetentionService : BackgroundService +{ + private readonly IMongoCollection _documents; + private readonly IMongoCollection _dtos; + private readonly GridFSBucket _bucket; + private readonly MongoStorageOptions _options; + private readonly ILogger _logger; + private readonly TimeProvider _timeProvider; + + public RawDocumentRetentionService( + IMongoDatabase database, + IOptions options, + ILogger logger, + TimeProvider? timeProvider = null) + { + ArgumentNullException.ThrowIfNull(database); + ArgumentNullException.ThrowIfNull(options); + ArgumentNullException.ThrowIfNull(logger); + + _documents = database.GetCollection(MongoStorageDefaults.Collections.Document); + _dtos = database.GetCollection(MongoStorageDefaults.Collections.Dto); + _bucket = new GridFSBucket(database, new GridFSBucketOptions + { + BucketName = "documents", + ReadConcern = database.Settings.ReadConcern, + WriteConcern = database.Settings.WriteConcern, + }); + + _options = options.Value; + _logger = logger; + _timeProvider = timeProvider ?? TimeProvider.System; + } + + protected override async Task ExecuteAsync(CancellationToken stoppingToken) + { + if (_options.RawDocumentRetention <= TimeSpan.Zero) + { + _logger.LogInformation("Raw document retention disabled; purge service idle."); + return; + } + + var sweepInterval = _options.RawDocumentRetentionSweepInterval > TimeSpan.Zero + ? _options.RawDocumentRetentionSweepInterval + : TimeSpan.FromHours(6); + + while (!stoppingToken.IsCancellationRequested) + { + try + { + await SweepExpiredDocumentsAsync(stoppingToken).ConfigureAwait(false); + } + catch (OperationCanceledException) when (stoppingToken.IsCancellationRequested) + { + break; + } + catch (Exception ex) + { + _logger.LogError(ex, "Raw document retention sweep failed"); + } + + try + { + await Task.Delay(sweepInterval, stoppingToken).ConfigureAwait(false); + } + catch (OperationCanceledException) when (stoppingToken.IsCancellationRequested) + { + break; + } + } + } + + internal async Task SweepExpiredDocumentsAsync(CancellationToken cancellationToken) + { + var grace = _options.RawDocumentRetentionTtlGrace >= TimeSpan.Zero + ? _options.RawDocumentRetentionTtlGrace + : TimeSpan.Zero; + var threshold = _timeProvider.GetUtcNow() + grace; + + var filterBuilder = Builders.Filter; + var filter = filterBuilder.And( + filterBuilder.Ne(doc => doc.ExpiresAt, null), + filterBuilder.Lte(doc => doc.ExpiresAt, threshold.UtcDateTime)); + + var removed = 0; + + while (!cancellationToken.IsCancellationRequested) + { + var batch = await _documents + .Find(filter) + .SortBy(doc => doc.ExpiresAt) + .Limit(200) + .ToListAsync(cancellationToken) + .ConfigureAwait(false); + + if (batch.Count == 0) + { + break; + } + + foreach (var document in batch) + { + if (cancellationToken.IsCancellationRequested) + { + break; + } + + await PurgeDocumentAsync(document, cancellationToken).ConfigureAwait(false); + removed++; + } + } + + if (removed > 0) + { + _logger.LogInformation("Purged {Count} expired raw documents (threshold <= {Threshold})", removed, threshold); + } + + return removed; + } + + private async Task PurgeDocumentAsync(DocumentDocument document, CancellationToken cancellationToken) + { + if (document.GridFsId.HasValue) + { + try + { + await _bucket.DeleteAsync(document.GridFsId.Value, cancellationToken).ConfigureAwait(false); + } + catch (GridFSFileNotFoundException) + { + // already removed or TTL swept + } + catch (Exception ex) + { + _logger.LogWarning(ex, "Failed to delete GridFS payload {GridFsId} for document {DocumentId}", document.GridFsId, document.Id); + } + } + + await _dtos.DeleteManyAsync(x => x.DocumentId == document.Id, cancellationToken).ConfigureAwait(false); + await _documents.DeleteOneAsync(x => x.Id == document.Id, cancellationToken).ConfigureAwait(false); + } +} diff --git a/src/StellaOps.Feedser.Storage.Mongo/ServiceCollectionExtensions.cs b/src/StellaOps.Feedser.Storage.Mongo/ServiceCollectionExtensions.cs new file mode 100644 index 00000000..27f28c74 --- /dev/null +++ b/src/StellaOps.Feedser.Storage.Mongo/ServiceCollectionExtensions.cs @@ -0,0 +1,90 @@ +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.DependencyInjection.Extensions; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Options; +using MongoDB.Driver; +using StellaOps.Feedser.Core.Jobs; +using StellaOps.Feedser.Storage.Mongo.Advisories; +using StellaOps.Feedser.Storage.Mongo.Aliases; +using StellaOps.Feedser.Storage.Mongo.ChangeHistory; +using StellaOps.Feedser.Storage.Mongo.Documents; +using StellaOps.Feedser.Storage.Mongo.Dtos; +using StellaOps.Feedser.Storage.Mongo.Exporting; +using StellaOps.Feedser.Storage.Mongo.JpFlags; +using StellaOps.Feedser.Storage.Mongo.MergeEvents; +using StellaOps.Feedser.Storage.Mongo.PsirtFlags; +using StellaOps.Feedser.Storage.Mongo.Migrations; + +namespace StellaOps.Feedser.Storage.Mongo; + +public static class ServiceCollectionExtensions +{ + public static IServiceCollection AddMongoStorage(this IServiceCollection services, Action configureOptions) + { + ArgumentNullException.ThrowIfNull(services); + ArgumentNullException.ThrowIfNull(configureOptions); + + services.AddOptions() + .Configure(configureOptions) + .PostConfigure(static options => options.EnsureValid()); + + services.TryAddSingleton(TimeProvider.System); + + services.AddSingleton(static sp => + { + var options = sp.GetRequiredService>().Value; + return new MongoClient(options.ConnectionString); + }); + + services.AddSingleton(static sp => + { + var options = sp.GetRequiredService>().Value; + var client = sp.GetRequiredService(); + var settings = new MongoDatabaseSettings + { + ReadConcern = ReadConcern.Majority, + WriteConcern = WriteConcern.WMajority, + ReadPreference = ReadPreference.PrimaryPreferred, + }; + + var database = client.GetDatabase(options.GetDatabaseName(), settings); + var writeConcern = database.Settings.WriteConcern.With(wTimeout: options.CommandTimeout); + return database.WithWriteConcern(writeConcern); + }); + + services.AddSingleton(); + services.AddSingleton(); + services.AddSingleton(); + services.AddSingleton(); + services.AddSingleton(); + services.AddSingleton(); + services.AddSingleton(); + services.AddSingleton(); + services.AddSingleton(); + services.AddSingleton(); + services.AddSingleton(); + services.AddSingleton(); + services.AddSingleton(); + services.TryAddSingleton(); + + services.AddSingleton>(static sp => + { + var database = sp.GetRequiredService(); + return database.GetCollection(MongoStorageDefaults.Collections.Jobs); + }); + + services.AddSingleton>(static sp => + { + var database = sp.GetRequiredService(); + return database.GetCollection(MongoStorageDefaults.Collections.Locks); + }); + + services.AddHostedService(); + + services.AddSingleton(); + services.AddSingleton(); + services.AddSingleton(); + + return services; + } +} diff --git a/src/StellaOps.Feedser.Storage.Mongo/SourceStateDocument.cs b/src/StellaOps.Feedser.Storage.Mongo/SourceStateDocument.cs new file mode 100644 index 00000000..d9dacfac --- /dev/null +++ b/src/StellaOps.Feedser.Storage.Mongo/SourceStateDocument.cs @@ -0,0 +1,73 @@ +using MongoDB.Bson; +using MongoDB.Bson.Serialization.Attributes; + +namespace StellaOps.Feedser.Storage.Mongo; + +[BsonIgnoreExtraElements] +public sealed class SourceStateDocument +{ + [BsonId] + public string SourceName { get; set; } = string.Empty; + + [BsonElement("enabled")] + public bool Enabled { get; set; } = true; + + [BsonElement("paused")] + public bool Paused { get; set; } + + [BsonElement("cursor")] + public BsonDocument Cursor { get; set; } = new(); + + [BsonElement("lastSuccess")] + [BsonIgnoreIfNull] + public DateTime? LastSuccess { get; set; } + + [BsonElement("lastFailure")] + [BsonIgnoreIfNull] + public DateTime? LastFailure { get; set; } + + [BsonElement("failCount")] + public int FailCount { get; set; } + + [BsonElement("backoffUntil")] + [BsonIgnoreIfNull] + public DateTime? BackoffUntil { get; set; } + + [BsonElement("updatedAt")] + public DateTime UpdatedAt { get; set; } + + [BsonElement("lastFailureReason")] + [BsonIgnoreIfNull] + public string? LastFailureReason { get; set; } +} + +internal static class SourceStateDocumentExtensions +{ + public static SourceStateDocument FromRecord(SourceStateRecord record) + => new() + { + SourceName = record.SourceName, + Enabled = record.Enabled, + Paused = record.Paused, + Cursor = record.Cursor ?? new BsonDocument(), + LastSuccess = record.LastSuccess?.UtcDateTime, + LastFailure = record.LastFailure?.UtcDateTime, + FailCount = record.FailCount, + BackoffUntil = record.BackoffUntil?.UtcDateTime, + UpdatedAt = record.UpdatedAt.UtcDateTime, + LastFailureReason = record.LastFailureReason, + }; + + public static SourceStateRecord ToRecord(this SourceStateDocument document) + => new( + document.SourceName, + document.Enabled, + document.Paused, + document.Cursor ?? new BsonDocument(), + document.LastSuccess.HasValue ? DateTime.SpecifyKind(document.LastSuccess.Value, DateTimeKind.Utc) : null, + document.LastFailure.HasValue ? DateTime.SpecifyKind(document.LastFailure.Value, DateTimeKind.Utc) : null, + document.FailCount, + document.BackoffUntil.HasValue ? DateTime.SpecifyKind(document.BackoffUntil.Value, DateTimeKind.Utc) : null, + DateTime.SpecifyKind(document.UpdatedAt, DateTimeKind.Utc), + document.LastFailureReason); +} diff --git a/src/StellaOps.Feedser.Storage.Mongo/SourceStateRecord.cs b/src/StellaOps.Feedser.Storage.Mongo/SourceStateRecord.cs new file mode 100644 index 00000000..6ea2f339 --- /dev/null +++ b/src/StellaOps.Feedser.Storage.Mongo/SourceStateRecord.cs @@ -0,0 +1,15 @@ +using MongoDB.Bson; + +namespace StellaOps.Feedser.Storage.Mongo; + +public sealed record SourceStateRecord( + string SourceName, + bool Enabled, + bool Paused, + BsonDocument Cursor, + DateTimeOffset? LastSuccess, + DateTimeOffset? LastFailure, + int FailCount, + DateTimeOffset? BackoffUntil, + DateTimeOffset UpdatedAt, + string? LastFailureReason); diff --git a/src/StellaOps.Feedser.Storage.Mongo/SourceStateRepositoryExtensions.cs b/src/StellaOps.Feedser.Storage.Mongo/SourceStateRepositoryExtensions.cs new file mode 100644 index 00000000..7d4bb300 --- /dev/null +++ b/src/StellaOps.Feedser.Storage.Mongo/SourceStateRepositoryExtensions.cs @@ -0,0 +1,19 @@ +using System; +using System.Threading; +using System.Threading.Tasks; + +namespace StellaOps.Feedser.Storage.Mongo; + +public static class SourceStateRepositoryExtensions +{ + public static Task MarkFailureAsync( + this ISourceStateRepository repository, + string sourceName, + DateTimeOffset failedAt, + TimeSpan? backoff, + CancellationToken cancellationToken) + { + ArgumentNullException.ThrowIfNull(repository); + return repository.MarkFailureAsync(sourceName, failedAt, backoff, failureReason: null, cancellationToken); + } +} diff --git a/src/StellaOps.Feedser.Storage.Mongo/StellaOps.Feedser.Storage.Mongo.csproj b/src/StellaOps.Feedser.Storage.Mongo/StellaOps.Feedser.Storage.Mongo.csproj new file mode 100644 index 00000000..e39dc64a --- /dev/null +++ b/src/StellaOps.Feedser.Storage.Mongo/StellaOps.Feedser.Storage.Mongo.csproj @@ -0,0 +1,19 @@ + + + net10.0 + preview + enable + enable + true + + + + + + + + + + + + diff --git a/src/StellaOps.Feedser.Storage.Mongo/TASKS.md b/src/StellaOps.Feedser.Storage.Mongo/TASKS.md new file mode 100644 index 00000000..5bfa9333 --- /dev/null +++ b/src/StellaOps.Feedser.Storage.Mongo/TASKS.md @@ -0,0 +1,19 @@ +# TASKS +| Task | Owner(s) | Depends on | Notes | +|---|---|---|---| +|MongoBootstrapper to create collections/indexes|BE-Storage|Storage.Mongo|DONE – `MongoBootstrapper` ensures collections & indexes incl. TTL on locks.ttlAt.| +|SourceState repository (get/set/backoff)|BE-Conn-Base|Storage.Mongo|DONE – implemented `MongoSourceStateRepository`.| +|Document/DTO stores with SHA/metadata|BE-Conn-Base|Storage.Mongo|DONE – DocumentStore and DtoStore provide upsert/status lookups.| +|AdvisoryStore (GetAllAsync etc.)|BE-Export|Models|DONE – AdvisoryStore handles upsert + recent/advisory fetches.| +|Job store (runs/active/recent)|BE-Core|Storage.Mongo|DONE – `MongoJobStore` covers create/start/complete queries.| +|Alias and reference secondary indexes|BE-Storage|Models|DONE – bootstrapper builds alias/reference indexes.| +|MergeEvent store|BE-Merge|Models|DONE – MergeEventStore appends/retrieves recent events.| +|ExportState store|BE-Export|Exporters|DONE – ExportStateStore upserts and retrieves exporter metadata.| +|Performance tests for large advisories|QA|Storage.Mongo|DONE – `AdvisoryStorePerformanceTests` exercises large payload upsert/find throughput budgets.| +|Migration playbook for schema/index changes|BE-Storage|Storage.Mongo|DONE – `MongoMigrationRunner` executes `IMongoMigration` steps recorded in `schema_migrations`; see `MIGRATIONS.md`.| +|Raw document retention/TTL strategy|BE-Storage|Storage.Mongo|DONE – retention options flow into `RawDocumentRetentionService` and TTL migrations for `document`/GridFS indexes.| +|Persist last failure reason in SourceState|BE-Storage|Storage.Mongo|DONE – `MongoSourceStateRepository.MarkFailureAsync` stores `lastFailureReason` with length guard + reset on success.| +|AdvisoryStore range primitives deserialization|BE-Storage|Models|DONE – BSON helpers handle `RangePrimitives`; regression test covers SemVer/NEVRA/EVR envelopes persisted through Mongo.| +|FEEDSTORAGE-DATA-03-001 Merge event provenance audit prep|BE-Storage|Merge|DONE – merge events now persist field-level decision reasons via `MergeFieldDecision` documents for analytics. **Coordination:** log any new precedence signals to storage@ so indexes/serializers stay aligned.| +|FEEDSTORAGE-DATA-02-001 Normalized range dual-write + backfill|BE-Storage|Core|DONE – advisory documents store `normalizedVersions`, migration respects `EnableSemVerStyle`, and decision reasons flow into normalized write path. **Action:** connector owners confirm `EnableSemVerStyle=true` readiness before 2025-10-18 rollout.| +|FEEDSTORAGE-TESTS-02-004 Restore AdvisoryStore build after normalized versions refactor|QA|Storage.Mongo|DONE – storage tests updated to cover normalized version payloads and new provenance fields. **Heads-up:** QA to watch for fixture bumps touching normalized rule arrays when connectors roll out support.| diff --git a/src/StellaOps.Feedser.Testing/ConnectorTestHarness.cs b/src/StellaOps.Feedser.Testing/ConnectorTestHarness.cs new file mode 100644 index 00000000..d2494f98 --- /dev/null +++ b/src/StellaOps.Feedser.Testing/ConnectorTestHarness.cs @@ -0,0 +1,118 @@ +using System; +using System.Linq; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Http; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Logging.Abstractions; +using StellaOps.Feedser.Source.Common.Http; +using Microsoft.Extensions.Time.Testing; +using StellaOps.Feedser.Source.Common; +using StellaOps.Feedser.Source.Common.Testing; +using StellaOps.Feedser.Storage.Mongo; +using StellaOps.Feedser.Testing; + +namespace StellaOps.Feedser.Testing; + +/// +/// Provides a reusable container for connector integration tests with canned HTTP responses and Mongo isolation. +/// +public sealed class ConnectorTestHarness : IAsyncDisposable +{ + private readonly MongoIntegrationFixture _fixture; + private readonly DateTimeOffset _initialTime; + private readonly string[] _httpClientNames; + private ServiceProvider? _serviceProvider; + + public ConnectorTestHarness(MongoIntegrationFixture fixture, DateTimeOffset initialTime, params string[] httpClientNames) + { + _fixture = fixture ?? throw new ArgumentNullException(nameof(fixture)); + _initialTime = initialTime; + _httpClientNames = httpClientNames.Length == 0 + ? Array.Empty() + : httpClientNames.Distinct(StringComparer.Ordinal).ToArray(); + + TimeProvider = new FakeTimeProvider(initialTime) + { + AutoAdvanceAmount = TimeSpan.Zero, + }; + Handler = new CannedHttpMessageHandler(); + } + + public FakeTimeProvider TimeProvider { get; } + + public CannedHttpMessageHandler Handler { get; } + + public ServiceProvider ServiceProvider => _serviceProvider ?? throw new InvalidOperationException("Call EnsureServiceProviderAsync first."); + + public async Task EnsureServiceProviderAsync(Action configureServices) + { + ArgumentNullException.ThrowIfNull(configureServices); + + if (_serviceProvider is not null) + { + return _serviceProvider; + } + + var services = new ServiceCollection(); + services.AddLogging(builder => builder.AddProvider(NullLoggerProvider.Instance)); + services.AddSingleton(TimeProvider); + services.AddSingleton(Handler); + + services.AddMongoStorage(options => + { + options.ConnectionString = _fixture.Runner.ConnectionString; + options.DatabaseName = _fixture.Database.DatabaseNamespace.DatabaseName; + options.CommandTimeout = TimeSpan.FromSeconds(5); + }); + + services.AddSourceCommon(); + + configureServices(services); + + foreach (var clientName in _httpClientNames) + { + services.Configure(clientName, options => + { + options.HttpMessageHandlerBuilderActions.Add(builder => + { + builder.PrimaryHandler = Handler; + }); + }); + } + + var provider = services.BuildServiceProvider(); + _serviceProvider = provider; + + var bootstrapper = provider.GetRequiredService(); + await bootstrapper.InitializeAsync(CancellationToken.None); + return provider; + } + + public async Task ResetAsync() + { + if (_serviceProvider is { } provider) + { + if (provider is IAsyncDisposable asyncDisposable) + { + await asyncDisposable.DisposeAsync(); + } + else + { + provider.Dispose(); + } + + _serviceProvider = null; + } + + await _fixture.Client.DropDatabaseAsync(_fixture.Database.DatabaseNamespace.DatabaseName); + Handler.Clear(); + TimeProvider.SetUtcNow(_initialTime); + } + + public async ValueTask DisposeAsync() + { + await ResetAsync(); + } +} diff --git a/src/StellaOps.Feedser.Testing/MongoIntegrationFixture.cs b/src/StellaOps.Feedser.Testing/MongoIntegrationFixture.cs new file mode 100644 index 00000000..c782edf8 --- /dev/null +++ b/src/StellaOps.Feedser.Testing/MongoIntegrationFixture.cs @@ -0,0 +1,27 @@ +using MongoDB.Bson; +using Mongo2Go; +using Xunit; +using MongoDB.Driver; + +namespace StellaOps.Feedser.Testing; + +public sealed class MongoIntegrationFixture : IAsyncLifetime +{ + public MongoDbRunner Runner { get; private set; } = null!; + public IMongoDatabase Database { get; private set; } = null!; + public IMongoClient Client { get; private set; } = null!; + + public Task InitializeAsync() + { + Runner = MongoDbRunner.Start(singleNodeReplSet: true); + Client = new MongoClient(Runner.ConnectionString); + Database = Client.GetDatabase($"feedser-tests-{Guid.NewGuid():N}"); + return Task.CompletedTask; + } + + public Task DisposeAsync() + { + Runner.Dispose(); + return Task.CompletedTask; + } +} diff --git a/src/StellaOps.Feedser.Testing/StellaOps.Feedser.Testing.csproj b/src/StellaOps.Feedser.Testing/StellaOps.Feedser.Testing.csproj new file mode 100644 index 00000000..2a35b8de --- /dev/null +++ b/src/StellaOps.Feedser.Testing/StellaOps.Feedser.Testing.csproj @@ -0,0 +1,20 @@ + + + net10.0 + enable + enable + true + false + + + + + + all + + + + + + + diff --git a/src/StellaOps.Feedser.Tests.Shared/AssemblyInfo.cs b/src/StellaOps.Feedser.Tests.Shared/AssemblyInfo.cs new file mode 100644 index 00000000..e43661c3 --- /dev/null +++ b/src/StellaOps.Feedser.Tests.Shared/AssemblyInfo.cs @@ -0,0 +1,3 @@ +using Xunit; + +[assembly: CollectionBehavior(DisableTestParallelization = true)] diff --git a/src/StellaOps.Feedser.Tests.Shared/MongoFixtureCollection.cs b/src/StellaOps.Feedser.Tests.Shared/MongoFixtureCollection.cs new file mode 100644 index 00000000..11610647 --- /dev/null +++ b/src/StellaOps.Feedser.Tests.Shared/MongoFixtureCollection.cs @@ -0,0 +1,6 @@ +using Xunit; + +namespace StellaOps.Feedser.Testing; + +[CollectionDefinition("mongo-fixture", DisableParallelization = true)] +public sealed class MongoFixtureCollection : ICollectionFixture; diff --git a/src/StellaOps.Feedser.WebService.Tests/FeedserOptionsPostConfigureTests.cs b/src/StellaOps.Feedser.WebService.Tests/FeedserOptionsPostConfigureTests.cs new file mode 100644 index 00000000..9c6ebef8 --- /dev/null +++ b/src/StellaOps.Feedser.WebService.Tests/FeedserOptionsPostConfigureTests.cs @@ -0,0 +1,56 @@ +using System; +using System.IO; +using StellaOps.Feedser.WebService.Options; +using Xunit; + +namespace StellaOps.Feedser.WebService.Tests; + +public sealed class FeedserOptionsPostConfigureTests +{ + [Fact] + public void Apply_LoadsClientSecretFromRelativeFile() + { + var tempDirectory = Directory.CreateTempSubdirectory(); + try + { + var secretPath = Path.Combine(tempDirectory.FullName, "authority.secret"); + File.WriteAllText(secretPath, " feedser-secret "); + + var options = new FeedserOptions + { + Authority = new FeedserOptions.AuthorityOptions + { + ClientSecretFile = "authority.secret" + } + }; + + FeedserOptionsPostConfigure.Apply(options, tempDirectory.FullName); + + Assert.Equal("feedser-secret", options.Authority.ClientSecret); + } + finally + { + if (Directory.Exists(tempDirectory.FullName)) + { + Directory.Delete(tempDirectory.FullName, recursive: true); + } + } + } + + [Fact] + public void Apply_ThrowsWhenSecretFileMissing() + { + var options = new FeedserOptions + { + Authority = new FeedserOptions.AuthorityOptions + { + ClientSecretFile = "missing.secret" + } + }; + + var exception = Assert.Throws(() => + FeedserOptionsPostConfigure.Apply(options, AppContext.BaseDirectory)); + + Assert.Contains("Authority client secret file", exception.Message); + } +} diff --git a/src/StellaOps.Feedser.WebService.Tests/PluginLoaderTests.cs b/src/StellaOps.Feedser.WebService.Tests/PluginLoaderTests.cs new file mode 100644 index 00000000..b089b983 --- /dev/null +++ b/src/StellaOps.Feedser.WebService.Tests/PluginLoaderTests.cs @@ -0,0 +1,29 @@ +using StellaOps.Plugin; + +namespace StellaOps.Feedser.WebService.Tests; + +public class PluginLoaderTests +{ + private sealed class NullServices : IServiceProvider + { + public object? GetService(Type serviceType) => null; + } + + [Fact] + public void ScansConnectorPluginsDirectory() + { + var services = new NullServices(); + var catalog = new PluginCatalog().AddFromDirectory(Path.Combine(AppContext.BaseDirectory, "PluginBinaries")); + var plugins = catalog.GetAvailableConnectorPlugins(services); + Assert.NotNull(plugins); + } + + [Fact] + public void ScansExporterPluginsDirectory() + { + var services = new NullServices(); + var catalog = new PluginCatalog().AddFromDirectory(Path.Combine(AppContext.BaseDirectory, "PluginBinaries")); + var plugins = catalog.GetAvailableExporterPlugins(services); + Assert.NotNull(plugins); + } +} diff --git a/src/StellaOps.Feedser.WebService.Tests/StellaOps.Feedser.WebService.Tests.csproj b/src/StellaOps.Feedser.WebService.Tests/StellaOps.Feedser.WebService.Tests.csproj new file mode 100644 index 00000000..51c5b3b7 --- /dev/null +++ b/src/StellaOps.Feedser.WebService.Tests/StellaOps.Feedser.WebService.Tests.csproj @@ -0,0 +1,13 @@ + + + net10.0 + enable + enable + + + + + + + + diff --git a/src/StellaOps.Feedser.WebService.Tests/WebServiceEndpointsTests.cs b/src/StellaOps.Feedser.WebService.Tests/WebServiceEndpointsTests.cs new file mode 100644 index 00000000..c6718432 --- /dev/null +++ b/src/StellaOps.Feedser.WebService.Tests/WebServiceEndpointsTests.cs @@ -0,0 +1,652 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Net; +using System.Net.Http.Json; +using Microsoft.AspNetCore.Builder; +using Microsoft.AspNetCore.Hosting; +using Microsoft.AspNetCore.Mvc.Testing; +using Microsoft.AspNetCore.TestHost; +using Microsoft.Extensions.Configuration; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Options; +using Mongo2Go; +using StellaOps.Feedser.Core.Jobs; +using StellaOps.Feedser.WebService.Jobs; +using StellaOps.Feedser.WebService.Options; +using Xunit.Sdk; +using StellaOps.Auth.Abstractions; + +namespace StellaOps.Feedser.WebService.Tests; + +public sealed class WebServiceEndpointsTests : IAsyncLifetime +{ + private MongoDbRunner _runner = null!; + private FeedserApplicationFactory _factory = null!; + + public Task InitializeAsync() + { + _runner = MongoDbRunner.Start(singleNodeReplSet: true); + _factory = new FeedserApplicationFactory(_runner.ConnectionString); + return Task.CompletedTask; + } + + public Task DisposeAsync() + { + _factory.Dispose(); + _runner.Dispose(); + return Task.CompletedTask; + } + + [Fact] + public async Task HealthAndReadyEndpointsRespond() + { + using var client = _factory.CreateClient(); + + var healthResponse = await client.GetAsync("/health"); + if (!healthResponse.IsSuccessStatusCode) + { + var body = await healthResponse.Content.ReadAsStringAsync(); + throw new Xunit.Sdk.XunitException($"/health failed: {(int)healthResponse.StatusCode} {body}"); + } + + var readyResponse = await client.GetAsync("/ready"); + if (!readyResponse.IsSuccessStatusCode) + { + var body = await readyResponse.Content.ReadAsStringAsync(); + throw new Xunit.Sdk.XunitException($"/ready failed: {(int)readyResponse.StatusCode} {body}"); + } + + var healthPayload = await healthResponse.Content.ReadFromJsonAsync(); + Assert.NotNull(healthPayload); + Assert.Equal("healthy", healthPayload!.Status); + Assert.Equal("mongo", healthPayload.Storage.Driver); + + var readyPayload = await readyResponse.Content.ReadFromJsonAsync(); + Assert.NotNull(readyPayload); + Assert.Equal("ready", readyPayload!.Status); + Assert.Equal("ready", readyPayload.Mongo.Status); + } + + [Fact] + public async Task JobsEndpointsReturnExpectedStatuses() + { + using var client = _factory.CreateClient(); + + var definitions = await client.GetAsync("/jobs/definitions"); + if (!definitions.IsSuccessStatusCode) + { + var body = await definitions.Content.ReadAsStringAsync(); + throw new Xunit.Sdk.XunitException($"/jobs/definitions failed: {(int)definitions.StatusCode} {body}"); + } + + var trigger = await client.PostAsync("/jobs/unknown", new StringContent("{}", System.Text.Encoding.UTF8, "application/json")); + if (trigger.StatusCode != HttpStatusCode.NotFound) + { + var payload = await trigger.Content.ReadAsStringAsync(); + throw new Xunit.Sdk.XunitException($"/jobs/unknown expected 404, got {(int)trigger.StatusCode}: {payload}"); + } + var problem = await trigger.Content.ReadFromJsonAsync(); + Assert.NotNull(problem); + Assert.Equal("https://stellaops.org/problems/not-found", problem!.Type); + Assert.Equal(404, problem.Status); + } + + [Fact] + public async Task JobRunEndpointReturnsProblemWhenNotFound() + { + using var client = _factory.CreateClient(); + var response = await client.GetAsync($"/jobs/{Guid.NewGuid()}"); + if (response.StatusCode != HttpStatusCode.NotFound) + { + var body = await response.Content.ReadAsStringAsync(); + throw new Xunit.Sdk.XunitException($"/jobs/{{id}} expected 404, got {(int)response.StatusCode}: {body}"); + } + var problem = await response.Content.ReadFromJsonAsync(); + Assert.NotNull(problem); + Assert.Equal("https://stellaops.org/problems/not-found", problem!.Type); + } + + [Fact] + public async Task JobTriggerMapsCoordinatorOutcomes() + { + var handler = _factory.Services.GetRequiredService(); + using var client = _factory.CreateClient(); + + handler.NextResult = JobTriggerResult.AlreadyRunning("busy"); + var conflict = await client.PostAsync("/jobs/test", JsonContent.Create(new JobTriggerRequest())); + if (conflict.StatusCode != HttpStatusCode.Conflict) + { + var payload = await conflict.Content.ReadAsStringAsync(); + throw new Xunit.Sdk.XunitException($"Conflict path expected 409, got {(int)conflict.StatusCode}: {payload}"); + } + var conflictProblem = await conflict.Content.ReadFromJsonAsync(); + Assert.NotNull(conflictProblem); + Assert.Equal("https://stellaops.org/problems/conflict", conflictProblem!.Type); + + handler.NextResult = JobTriggerResult.Accepted(new JobRunSnapshot(Guid.NewGuid(), "demo", JobRunStatus.Pending, DateTimeOffset.UtcNow, null, null, "api", null, null, null, null, new Dictionary())); + var accepted = await client.PostAsync("/jobs/test", JsonContent.Create(new JobTriggerRequest())); + if (accepted.StatusCode != HttpStatusCode.Accepted) + { + var payload = await accepted.Content.ReadAsStringAsync(); + throw new Xunit.Sdk.XunitException($"Accepted path expected 202, got {(int)accepted.StatusCode}: {payload}"); + } + Assert.NotNull(accepted.Headers.Location); + var acceptedPayload = await accepted.Content.ReadFromJsonAsync(); + Assert.NotNull(acceptedPayload); + + handler.NextResult = JobTriggerResult.Failed(new JobRunSnapshot(Guid.NewGuid(), "demo", JobRunStatus.Failed, DateTimeOffset.UtcNow, null, DateTimeOffset.UtcNow, "api", null, "err", null, null, new Dictionary()), "boom"); + var failed = await client.PostAsync("/jobs/test", JsonContent.Create(new JobTriggerRequest())); + if (failed.StatusCode != HttpStatusCode.InternalServerError) + { + var payload = await failed.Content.ReadAsStringAsync(); + throw new Xunit.Sdk.XunitException($"Failed path expected 500, got {(int)failed.StatusCode}: {payload}"); + } + var failureProblem = await failed.Content.ReadFromJsonAsync(); + Assert.NotNull(failureProblem); + Assert.Equal("https://stellaops.org/problems/job-failure", failureProblem!.Type); + } + + [Fact] + public async Task JobsEndpointsExposeJobData() + { + var handler = _factory.Services.GetRequiredService(); + var now = DateTimeOffset.UtcNow; + var run = new JobRunSnapshot( + Guid.NewGuid(), + "demo", + JobRunStatus.Succeeded, + now, + now, + now.AddSeconds(2), + "api", + "hash", + null, + TimeSpan.FromMinutes(5), + TimeSpan.FromMinutes(1), + new Dictionary { ["key"] = "value" }); + + handler.Definitions = new[] + { + new JobDefinition("demo", typeof(DemoJob), TimeSpan.FromMinutes(5), TimeSpan.FromMinutes(1), "*/5 * * * *", true) + }; + handler.LastRuns["demo"] = run; + handler.RecentRuns = new[] { run }; + handler.ActiveRuns = Array.Empty(); + handler.Runs[run.RunId] = run; + + try + { + using var client = _factory.CreateClient(); + + var definitions = await client.GetFromJsonAsync>("/jobs/definitions"); + Assert.NotNull(definitions); + Assert.Single(definitions!); + Assert.Equal("demo", definitions![0].Kind); + Assert.NotNull(definitions[0].LastRun); + Assert.Equal(run.RunId, definitions[0].LastRun!.RunId); + + var runPayload = await client.GetFromJsonAsync($"/jobs/{run.RunId}"); + Assert.NotNull(runPayload); + Assert.Equal(run.RunId, runPayload!.RunId); + Assert.Equal("Succeeded", runPayload.Status); + + var runs = await client.GetFromJsonAsync>("/jobs?kind=demo&limit=5"); + Assert.NotNull(runs); + Assert.Single(runs!); + Assert.Equal(run.RunId, runs![0].RunId); + + var runsByDefinition = await client.GetFromJsonAsync>("/jobs/definitions/demo/runs"); + Assert.NotNull(runsByDefinition); + Assert.Single(runsByDefinition!); + + var active = await client.GetFromJsonAsync>("/jobs/active"); + Assert.NotNull(active); + Assert.Empty(active!); + } + finally + { + handler.Definitions = Array.Empty(); + handler.RecentRuns = Array.Empty(); + handler.ActiveRuns = Array.Empty(); + handler.Runs.Clear(); + handler.LastRuns.Clear(); + } + } + + [Fact] + public async Task JobsEndpointsAllowBypassWhenAuthorityEnabled() + { + var environment = new Dictionary + { + ["FEEDSER_AUTHORITY__ENABLED"] = "true", + ["FEEDSER_AUTHORITY__ALLOWANONYMOUSFALLBACK"] = "false", + ["FEEDSER_AUTHORITY__ISSUER"] = "https://authority.example", + ["FEEDSER_AUTHORITY__REQUIREHTTPSMETADATA"] = "false", + ["FEEDSER_AUTHORITY__AUDIENCES__0"] = "api://feedser", + ["FEEDSER_AUTHORITY__REQUIREDSCOPES__0"] = StellaOpsScopes.FeedserJobsTrigger, + ["FEEDSER_AUTHORITY__BYPASSNETWORKS__0"] = "127.0.0.1/32", + ["FEEDSER_AUTHORITY__BYPASSNETWORKS__1"] = "::1/128", + ["FEEDSER_AUTHORITY__CLIENTID"] = "feedser-jobs", + ["FEEDSER_AUTHORITY__CLIENTSECRET"] = "test-secret", + ["FEEDSER_AUTHORITY__CLIENTSCOPES__0"] = StellaOpsScopes.FeedserJobsTrigger, + }; + + using var factory = new FeedserApplicationFactory( + _runner.ConnectionString, + authority => + { + authority.Enabled = true; + authority.AllowAnonymousFallback = false; + authority.Issuer = "https://authority.example"; + authority.RequireHttpsMetadata = false; + authority.Audiences.Clear(); + authority.Audiences.Add("api://feedser"); + authority.RequiredScopes.Clear(); + authority.RequiredScopes.Add(StellaOpsScopes.FeedserJobsTrigger); + authority.BypassNetworks.Clear(); + authority.BypassNetworks.Add("127.0.0.1/32"); + authority.BypassNetworks.Add("::1/128"); + authority.ClientId = "feedser-jobs"; + authority.ClientSecret = "test-secret"; + }, + environment); + + var handler = factory.Services.GetRequiredService(); + handler.Definitions = new[] { new JobDefinition("demo", typeof(DemoJob), TimeSpan.FromMinutes(5), TimeSpan.FromMinutes(1), null, true) }; + + using var client = factory.CreateClient(); + client.DefaultRequestHeaders.Add("X-Test-RemoteAddr", "127.0.0.1"); + var response = await client.GetAsync("/jobs/definitions"); + + Assert.Equal(HttpStatusCode.OK, response.StatusCode); + + var auditLogs = factory.LoggerProvider.Snapshot("Feedser.Authorization.Audit"); + var bypassLog = Assert.Single(auditLogs, entry => entry.TryGetState("Bypass", out var state) && state is bool flag && flag); + Assert.True(bypassLog.TryGetState("RemoteAddress", out var remoteObj) && string.Equals(remoteObj?.ToString(), "127.0.0.1", StringComparison.Ordinal)); + Assert.True(bypassLog.TryGetState("StatusCode", out var statusObj) && Convert.ToInt32(statusObj) == (int)HttpStatusCode.OK); + } + + [Fact] + public async Task JobsEndpointsRequireAuthWhenFallbackDisabled() + { + var enforcementEnvironment = new Dictionary + { + ["FEEDSER_AUTHORITY__ENABLED"] = "true", + ["FEEDSER_AUTHORITY__ALLOWANONYMOUSFALLBACK"] = "false", + ["FEEDSER_AUTHORITY__ISSUER"] = "https://authority.example", + ["FEEDSER_AUTHORITY__REQUIREHTTPSMETADATA"] = "false", + ["FEEDSER_AUTHORITY__AUDIENCES__0"] = "api://feedser", + ["FEEDSER_AUTHORITY__REQUIREDSCOPES__0"] = StellaOpsScopes.FeedserJobsTrigger, + ["FEEDSER_AUTHORITY__CLIENTID"] = "feedser-jobs", + ["FEEDSER_AUTHORITY__CLIENTSECRET"] = "test-secret", + ["FEEDSER_AUTHORITY__CLIENTSCOPES__0"] = StellaOpsScopes.FeedserJobsTrigger, + }; + + using var factory = new FeedserApplicationFactory( + _runner.ConnectionString, + authority => + { + authority.Enabled = true; + authority.AllowAnonymousFallback = false; + authority.Issuer = "https://authority.example"; + authority.RequireHttpsMetadata = false; + authority.Audiences.Clear(); + authority.Audiences.Add("api://feedser"); + authority.RequiredScopes.Clear(); + authority.RequiredScopes.Add(StellaOpsScopes.FeedserJobsTrigger); + authority.BypassNetworks.Clear(); + authority.ClientId = "feedser-jobs"; + authority.ClientSecret = "test-secret"; + }, + enforcementEnvironment); + + var resolved = factory.Services.GetRequiredService>().Value; + Assert.False(resolved.Authority.AllowAnonymousFallback); + + using var client = factory.CreateClient(); + client.DefaultRequestHeaders.Add("X-Test-RemoteAddr", "127.0.0.1"); + var response = await client.GetAsync("/jobs/definitions"); + + Assert.Equal(HttpStatusCode.Unauthorized, response.StatusCode); + + var auditLogs = factory.LoggerProvider.Snapshot("Feedser.Authorization.Audit"); + var enforcementLog = Assert.Single(auditLogs); + Assert.True(enforcementLog.TryGetState("BypassAllowed", out var bypassAllowedObj) && bypassAllowedObj is bool bypassAllowed && bypassAllowed == false); + Assert.True(enforcementLog.TryGetState("HasPrincipal", out var principalObj) && principalObj is bool hasPrincipal && hasPrincipal == false); + } + + private sealed class FeedserApplicationFactory : WebApplicationFactory + { + private readonly string _connectionString; + private readonly string? _previousDsn; + private readonly string? _previousDriver; + private readonly string? _previousTimeout; + private readonly string? _previousTelemetryEnabled; + private readonly string? _previousTelemetryLogging; + private readonly string? _previousTelemetryTracing; + private readonly string? _previousTelemetryMetrics; + private readonly Action? _authorityConfigure; + private readonly IDictionary _additionalPreviousEnvironment = new Dictionary(StringComparer.OrdinalIgnoreCase); + public CollectingLoggerProvider LoggerProvider { get; } = new(); + + public FeedserApplicationFactory( + string connectionString, + Action? authorityConfigure = null, + IDictionary? environmentOverrides = null) + { + _connectionString = connectionString; + _authorityConfigure = authorityConfigure; + _previousDsn = Environment.GetEnvironmentVariable("FEEDSER_STORAGE__DSN"); + _previousDriver = Environment.GetEnvironmentVariable("FEEDSER_STORAGE__DRIVER"); + _previousTimeout = Environment.GetEnvironmentVariable("FEEDSER_STORAGE__COMMANDTIMEOUTSECONDS"); + _previousTelemetryEnabled = Environment.GetEnvironmentVariable("FEEDSER_TELEMETRY__ENABLED"); + _previousTelemetryLogging = Environment.GetEnvironmentVariable("FEEDSER_TELEMETRY__ENABLELOGGING"); + _previousTelemetryTracing = Environment.GetEnvironmentVariable("FEEDSER_TELEMETRY__ENABLETRACING"); + _previousTelemetryMetrics = Environment.GetEnvironmentVariable("FEEDSER_TELEMETRY__ENABLEMETRICS"); + Environment.SetEnvironmentVariable("FEEDSER_STORAGE__DSN", connectionString); + Environment.SetEnvironmentVariable("FEEDSER_STORAGE__DRIVER", "mongo"); + Environment.SetEnvironmentVariable("FEEDSER_STORAGE__COMMANDTIMEOUTSECONDS", "30"); + Environment.SetEnvironmentVariable("FEEDSER_TELEMETRY__ENABLED", "false"); + Environment.SetEnvironmentVariable("FEEDSER_TELEMETRY__ENABLELOGGING", "false"); + Environment.SetEnvironmentVariable("FEEDSER_TELEMETRY__ENABLETRACING", "false"); + Environment.SetEnvironmentVariable("FEEDSER_TELEMETRY__ENABLEMETRICS", "false"); + if (environmentOverrides is not null) + { + foreach (var kvp in environmentOverrides) + { + var previous = Environment.GetEnvironmentVariable(kvp.Key); + _additionalPreviousEnvironment[kvp.Key] = previous; + Environment.SetEnvironmentVariable(kvp.Key, kvp.Value); + } + } + } + + protected override void ConfigureWebHost(IWebHostBuilder builder) + { + builder.ConfigureAppConfiguration((context, configurationBuilder) => + { + var settings = new Dictionary + { + ["Plugins:Directory"] = Path.Combine(context.HostingEnvironment.ContentRootPath, "PluginBinaries"), + }; + + configurationBuilder.AddInMemoryCollection(settings!); + }); + + builder.ConfigureLogging(logging => + { + logging.AddProvider(LoggerProvider); + }); + + builder.ConfigureServices(services => + { + services.AddSingleton(); + services.AddSingleton(sp => sp.GetRequiredService()); + services.PostConfigure(options => + { + options.Storage.Driver = "mongo"; + options.Storage.Dsn = _connectionString; + options.Storage.CommandTimeoutSeconds = 30; + options.Plugins.Directory ??= Path.Combine(AppContext.BaseDirectory, "PluginBinaries"); + options.Telemetry.Enabled = false; + options.Telemetry.EnableLogging = false; + options.Telemetry.EnableTracing = false; + options.Telemetry.EnableMetrics = false; + options.Authority ??= new FeedserOptions.AuthorityOptions(); + _authorityConfigure?.Invoke(options.Authority); + }); + }); + + builder.ConfigureTestServices(services => + { + services.AddSingleton(); + }); + } + + protected override void Dispose(bool disposing) + { + base.Dispose(disposing); + Environment.SetEnvironmentVariable("FEEDSER_STORAGE__DSN", _previousDsn); + Environment.SetEnvironmentVariable("FEEDSER_STORAGE__DRIVER", _previousDriver); + Environment.SetEnvironmentVariable("FEEDSER_STORAGE__COMMANDTIMEOUTSECONDS", _previousTimeout); + Environment.SetEnvironmentVariable("FEEDSER_TELEMETRY__ENABLED", _previousTelemetryEnabled); + Environment.SetEnvironmentVariable("FEEDSER_TELEMETRY__ENABLELOGGING", _previousTelemetryLogging); + Environment.SetEnvironmentVariable("FEEDSER_TELEMETRY__ENABLETRACING", _previousTelemetryTracing); + Environment.SetEnvironmentVariable("FEEDSER_TELEMETRY__ENABLEMETRICS", _previousTelemetryMetrics); + foreach (var kvp in _additionalPreviousEnvironment) + { + Environment.SetEnvironmentVariable(kvp.Key, kvp.Value); + } + + LoggerProvider.Dispose(); + } + + private sealed class RemoteIpStartupFilter : IStartupFilter + { + public Action Configure(Action next) + { + return app => + { + app.Use(async (context, nextMiddleware) => + { + if (context.Request.Headers.TryGetValue("X-Test-RemoteAddr", out var values) + && values.Count > 0 + && IPAddress.TryParse(values[0], out var remote)) + { + context.Connection.RemoteIpAddress = remote; + } + + await nextMiddleware(); + }); + + next(app); + }; + } + } + + public sealed record LogEntry( + string LoggerName, + LogLevel Level, + EventId EventId, + string? Message, + Exception? Exception, + IReadOnlyList> State) + { + public bool TryGetState(string name, out object? value) + { + foreach (var kvp in State) + { + if (string.Equals(kvp.Key, name, StringComparison.Ordinal)) + { + value = kvp.Value; + return true; + } + } + + value = null; + return false; + } + } + + public sealed class CollectingLoggerProvider : ILoggerProvider + { + private readonly object syncRoot = new(); + private readonly List entries = new(); + private bool disposed; + + public ILogger CreateLogger(string categoryName) => new CollectingLogger(categoryName, this); + + public IReadOnlyList Snapshot(string loggerName) + { + lock (syncRoot) + { + return entries + .Where(entry => string.Equals(entry.LoggerName, loggerName, StringComparison.Ordinal)) + .ToArray(); + } + } + + public void Dispose() + { + disposed = true; + lock (syncRoot) + { + entries.Clear(); + } + } + + private void Append(LogEntry entry) + { + if (disposed) + { + return; + } + + lock (syncRoot) + { + entries.Add(entry); + } + } + + private sealed class CollectingLogger : ILogger + { + private readonly string categoryName; + private readonly CollectingLoggerProvider provider; + + public CollectingLogger(string categoryName, CollectingLoggerProvider provider) + { + this.categoryName = categoryName; + this.provider = provider; + } + + public IDisposable? BeginScope(TState state) where TState : notnull => NullScope.Instance; + + public bool IsEnabled(LogLevel logLevel) => true; + + public void Log(LogLevel logLevel, EventId eventId, TState state, Exception? exception, Func formatter) + { + if (formatter is null) + { + throw new ArgumentNullException(nameof(formatter)); + } + + var message = formatter(state, exception); + var kvps = ExtractState(state); + var entry = new LogEntry(categoryName, logLevel, eventId, message, exception, kvps); + provider.Append(entry); + } + + private static IReadOnlyList> ExtractState(TState state) + { + if (state is IReadOnlyList> list) + { + return list; + } + + if (state is IEnumerable> enumerable) + { + return enumerable.ToArray(); + } + + if (state is null) + { + return Array.Empty>(); + } + + return new[] { new KeyValuePair("State", state) }; + } + } + + private sealed class NullScope : IDisposable + { + public static readonly NullScope Instance = new(); + public void Dispose() + { + } + } + } + } + + private sealed record HealthPayload(string Status, DateTimeOffset StartedAt, double UptimeSeconds, StoragePayload Storage, TelemetryPayload Telemetry); + + private sealed record StoragePayload(string Driver, bool Completed, DateTimeOffset? CompletedAt, double? DurationMs); + + private sealed record TelemetryPayload(bool Enabled, bool Tracing, bool Metrics, bool Logging); + + private sealed record ReadyPayload(string Status, DateTimeOffset StartedAt, double UptimeSeconds, ReadyMongoPayload Mongo); + + private sealed record ReadyMongoPayload(string Status, double? LatencyMs, DateTimeOffset? CheckedAt, string? Error); + + private sealed record JobDefinitionPayload(string Kind, bool Enabled, string? CronExpression, TimeSpan Timeout, TimeSpan LeaseDuration, JobRunPayload? LastRun); + + private sealed record JobRunPayload(Guid RunId, string Kind, string Status, string Trigger, DateTimeOffset CreatedAt, DateTimeOffset? StartedAt, DateTimeOffset? CompletedAt, string? Error, TimeSpan? Duration, Dictionary Parameters); + + private sealed record ProblemDocument(string? Type, string? Title, int? Status, string? Detail, string? Instance); + + private sealed class DemoJob : IJob + { + public Task ExecuteAsync(JobExecutionContext context, CancellationToken cancellationToken) => Task.CompletedTask; + } + + private sealed class StubJobCoordinator : IJobCoordinator + { + public JobTriggerResult NextResult { get; set; } = JobTriggerResult.NotFound("not set"); + + public IReadOnlyList Definitions { get; set; } = Array.Empty(); + + public IReadOnlyList RecentRuns { get; set; } = Array.Empty(); + + public IReadOnlyList ActiveRuns { get; set; } = Array.Empty(); + + public Dictionary Runs { get; } = new(); + + public Dictionary LastRuns { get; } = new(StringComparer.Ordinal); + + public Task TriggerAsync(string kind, IReadOnlyDictionary? parameters, string trigger, CancellationToken cancellationToken) + => Task.FromResult(NextResult); + + public Task> GetDefinitionsAsync(CancellationToken cancellationToken) + => Task.FromResult(Definitions); + + public Task> GetRecentRunsAsync(string? kind, int limit, CancellationToken cancellationToken) + { + IEnumerable query = RecentRuns; + if (!string.IsNullOrWhiteSpace(kind)) + { + query = query.Where(run => string.Equals(run.Kind, kind, StringComparison.Ordinal)); + } + + return Task.FromResult>(query.Take(limit).ToArray()); + } + + public Task> GetActiveRunsAsync(CancellationToken cancellationToken) + => Task.FromResult(ActiveRuns); + + public Task GetRunAsync(Guid runId, CancellationToken cancellationToken) + => Task.FromResult(Runs.TryGetValue(runId, out var run) ? run : null); + + public Task GetLastRunAsync(string kind, CancellationToken cancellationToken) + => Task.FromResult(LastRuns.TryGetValue(kind, out var run) ? run : null); + + public Task> GetLastRunsAsync(IEnumerable kinds, CancellationToken cancellationToken) + { + var map = new Dictionary(StringComparer.Ordinal); + foreach (var kind in kinds) + { + if (kind is null) + { + continue; + } + + if (LastRuns.TryGetValue(kind, out var run) && run is not null) + { + map[kind] = run; + } + } + + return Task.FromResult>(map); + } + } +} diff --git a/src/StellaOps.Feedser.WebService/AGENTS.md b/src/StellaOps.Feedser.WebService/AGENTS.md new file mode 100644 index 00000000..c6bbeabf --- /dev/null +++ b/src/StellaOps.Feedser.WebService/AGENTS.md @@ -0,0 +1,34 @@ +# AGENTS +## Role +Minimal API host wiring configuration, storage, plugin routines, and job endpoints. Operational surface for health, readiness, and job control. +## Scope +- Configuration: appsettings.json + etc/feedser.yaml (yaml path = ../etc/feedser.yaml); bind into FeedserOptions with validation (Only Mongo supported). +- Mongo: MongoUrl from options.Storage.Dsn; IMongoClient/IMongoDatabase singletons; default database name fallback (options -> URL -> "feedser"). +- Services: AddMongoStorage(); AddSourceHttpClients(); RegisterPluginRoutines(configuration, PluginHostOptions). +- Bootstrap: MongoBootstrapper.InitializeAsync on startup. +- Endpoints (configuration & job control only; root path intentionally unbound): + - GET /health -> {status:"healthy"} after options validation binds. + - GET /ready -> MongoDB ping; 503 on MongoException/Timeout. + - GET /jobs?kind=&limit= -> recent runs. + - GET /jobs/{id} -> run detail. + - GET /jobs/definitions -> definitions with lastRun. + - GET /jobs/definitions/{kind} -> definition + lastRun or 404. + - GET /jobs/definitions/{kind}/runs?limit= -> recent runs or 404 if kind unknown. + - GET /jobs/active -> currently running. + - POST /jobs/{*jobKind} with {trigger?,parameters?} -> 202 Accepted (Location:/jobs/{runId}) | 404 | 409 | 423. +- PluginHost defaults: BaseDirectory = solution root; PluginsDirectory = "PluginBinaries"; SearchPatterns += "StellaOps.Feedser.Plugin.*.dll"; EnsureDirectoryExists = true. +## Participants +- Core job system; Storage.Mongo; Source.Common HTTP clients; Exporter and Connector plugin routines discover/register jobs. +## Interfaces & contracts +- Dependency injection boundary for all connectors/exporters; IOptions validated on start. +- Cancellation: pass app.Lifetime.ApplicationStopping to bootstrapper. +## In/Out of scope +In: hosting, DI composition, REST surface, readiness checks. +Out: business logic of jobs, HTML UI, authn/z (future). +## Observability & security expectations +- Log startup config (redact DSN credentials), plugin scan results (missing ordered plugins if any). +- Structured responses with status codes; no stack traces in HTTP bodies; errors mapped cleanly. +## Tests +- Author and review coverage in `../StellaOps.Feedser.WebService.Tests`. +- Shared fixtures (e.g., `MongoIntegrationFixture`, `ConnectorTestHarness`) live in `../StellaOps.Feedser.Testing`. +- Keep fixtures deterministic; match new cases to real-world advisories or regression scenarios. diff --git a/src/StellaOps.Feedser.WebService/Diagnostics/HealthContracts.cs b/src/StellaOps.Feedser.WebService/Diagnostics/HealthContracts.cs new file mode 100644 index 00000000..63707650 --- /dev/null +++ b/src/StellaOps.Feedser.WebService/Diagnostics/HealthContracts.cs @@ -0,0 +1,32 @@ +namespace StellaOps.Feedser.WebService.Diagnostics; + +internal sealed record StorageBootstrapHealth( + string Driver, + bool Completed, + DateTimeOffset? CompletedAt, + double? DurationMs); + +internal sealed record TelemetryHealth( + bool Enabled, + bool Tracing, + bool Metrics, + bool Logging); + +internal sealed record HealthDocument( + string Status, + DateTimeOffset StartedAt, + double UptimeSeconds, + StorageBootstrapHealth Storage, + TelemetryHealth Telemetry); + +internal sealed record MongoReadyHealth( + string Status, + double? LatencyMs, + DateTimeOffset? CheckedAt, + string? Error); + +internal sealed record ReadyDocument( + string Status, + DateTimeOffset StartedAt, + double UptimeSeconds, + MongoReadyHealth Mongo); diff --git a/src/StellaOps.Feedser.WebService/Diagnostics/JobMetrics.cs b/src/StellaOps.Feedser.WebService/Diagnostics/JobMetrics.cs new file mode 100644 index 00000000..cb85ff95 --- /dev/null +++ b/src/StellaOps.Feedser.WebService/Diagnostics/JobMetrics.cs @@ -0,0 +1,25 @@ +using System.Diagnostics.Metrics; + +namespace StellaOps.Feedser.WebService.Diagnostics; + +internal static class JobMetrics +{ + internal const string MeterName = "StellaOps.Feedser.WebService.Jobs"; + + private static readonly Meter Meter = new(MeterName); + + internal static readonly Counter TriggerCounter = Meter.CreateCounter( + "web.jobs.triggered", + unit: "count", + description: "Number of job trigger requests accepted by the web service."); + + internal static readonly Counter TriggerConflictCounter = Meter.CreateCounter( + "web.jobs.trigger.conflict", + unit: "count", + description: "Number of job trigger requests that resulted in conflicts or rejections."); + + internal static readonly Counter TriggerFailureCounter = Meter.CreateCounter( + "web.jobs.trigger.failed", + unit: "count", + description: "Number of job trigger requests that failed at runtime."); +} diff --git a/src/StellaOps.Feedser.WebService/Diagnostics/ProblemTypes.cs b/src/StellaOps.Feedser.WebService/Diagnostics/ProblemTypes.cs new file mode 100644 index 00000000..8be947b9 --- /dev/null +++ b/src/StellaOps.Feedser.WebService/Diagnostics/ProblemTypes.cs @@ -0,0 +1,12 @@ +namespace StellaOps.Feedser.WebService.Diagnostics; + +internal static class ProblemTypes +{ + public const string NotFound = "https://stellaops.org/problems/not-found"; + public const string Validation = "https://stellaops.org/problems/validation"; + public const string Conflict = "https://stellaops.org/problems/conflict"; + public const string Locked = "https://stellaops.org/problems/locked"; + public const string LeaseRejected = "https://stellaops.org/problems/lease-rejected"; + public const string JobFailure = "https://stellaops.org/problems/job-failure"; + public const string ServiceUnavailable = "https://stellaops.org/problems/service-unavailable"; +} diff --git a/src/StellaOps.Feedser.WebService/Diagnostics/ServiceStatus.cs b/src/StellaOps.Feedser.WebService/Diagnostics/ServiceStatus.cs new file mode 100644 index 00000000..256d09bf --- /dev/null +++ b/src/StellaOps.Feedser.WebService/Diagnostics/ServiceStatus.cs @@ -0,0 +1,74 @@ +using System.Diagnostics; + +namespace StellaOps.Feedser.WebService.Diagnostics; + +internal sealed class ServiceStatus +{ + private readonly TimeProvider _timeProvider; + private readonly DateTimeOffset _startedAt; + private readonly object _sync = new(); + + private DateTimeOffset? _bootstrapCompletedAt; + private TimeSpan? _bootstrapDuration; + private DateTimeOffset? _lastReadyCheckAt; + private TimeSpan? _lastMongoLatency; + private string? _lastMongoError; + private bool _lastReadySucceeded; + + public ServiceStatus(TimeProvider timeProvider) + { + _timeProvider = timeProvider ?? TimeProvider.System; + _startedAt = _timeProvider.GetUtcNow(); + } + + public ServiceHealthSnapshot CreateSnapshot() + { + lock (_sync) + { + return new ServiceHealthSnapshot( + CapturedAt: _timeProvider.GetUtcNow(), + StartedAt: _startedAt, + BootstrapCompletedAt: _bootstrapCompletedAt, + BootstrapDuration: _bootstrapDuration, + LastReadyCheckAt: _lastReadyCheckAt, + LastMongoLatency: _lastMongoLatency, + LastMongoError: _lastMongoError, + LastReadySucceeded: _lastReadySucceeded); + } + } + + public void MarkBootstrapCompleted(TimeSpan duration) + { + lock (_sync) + { + var completedAt = _timeProvider.GetUtcNow(); + _bootstrapCompletedAt = completedAt; + _bootstrapDuration = duration; + _lastReadySucceeded = true; + _lastMongoLatency = duration; + _lastMongoError = null; + _lastReadyCheckAt = completedAt; + } + } + + public void RecordMongoCheck(bool success, TimeSpan latency, string? error) + { + lock (_sync) + { + _lastReadySucceeded = success; + _lastMongoLatency = latency; + _lastMongoError = success ? null : error; + _lastReadyCheckAt = _timeProvider.GetUtcNow(); + } + } +} + +internal sealed record ServiceHealthSnapshot( + DateTimeOffset CapturedAt, + DateTimeOffset StartedAt, + DateTimeOffset? BootstrapCompletedAt, + TimeSpan? BootstrapDuration, + DateTimeOffset? LastReadyCheckAt, + TimeSpan? LastMongoLatency, + string? LastMongoError, + bool LastReadySucceeded); diff --git a/src/StellaOps.Feedser.WebService/Extensions/ConfigurationExtensions.cs b/src/StellaOps.Feedser.WebService/Extensions/ConfigurationExtensions.cs new file mode 100644 index 00000000..155d7a49 --- /dev/null +++ b/src/StellaOps.Feedser.WebService/Extensions/ConfigurationExtensions.cs @@ -0,0 +1,38 @@ +using System.Text; +using System.Text.Json; +using Microsoft.Extensions.Configuration; +using YamlDotNet.Serialization; +using YamlDotNet.Serialization.NamingConventions; + +namespace StellaOps.Feedser.WebService.Extensions; + +public static class ConfigurationExtensions +{ + public static IConfigurationBuilder AddFeedserYaml(this IConfigurationBuilder builder, string path) + { + if (builder is null) + { + throw new ArgumentNullException(nameof(builder)); + } + + if (string.IsNullOrWhiteSpace(path) || !File.Exists(path)) + { + return builder; + } + + var deserializer = new DeserializerBuilder() + .WithNamingConvention(CamelCaseNamingConvention.Instance) + .Build(); + + using var reader = File.OpenText(path); + var yamlObject = deserializer.Deserialize(reader); + if (yamlObject is null) + { + return builder; + } + + var json = JsonSerializer.Serialize(yamlObject); + var stream = new MemoryStream(Encoding.UTF8.GetBytes(json)); + return builder.AddJsonStream(stream); + } +} diff --git a/src/StellaOps.Feedser.WebService/Extensions/JobRegistrationExtensions.cs b/src/StellaOps.Feedser.WebService/Extensions/JobRegistrationExtensions.cs new file mode 100644 index 00000000..73920247 --- /dev/null +++ b/src/StellaOps.Feedser.WebService/Extensions/JobRegistrationExtensions.cs @@ -0,0 +1,98 @@ +using System; +using System.Collections.Generic; +using System.Globalization; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Options; +using StellaOps.Feedser.Core.Jobs; +using StellaOps.Feedser.Merge.Jobs; + +namespace StellaOps.Feedser.WebService.Extensions; + +internal static class JobRegistrationExtensions +{ + private sealed record BuiltInJob( + string Kind, + string JobType, + string AssemblyName, + TimeSpan Timeout, + TimeSpan LeaseDuration, + string? CronExpression = null); + + private static readonly IReadOnlyList BuiltInJobs = new List + { + new("source:redhat:fetch", "StellaOps.Feedser.Source.Distro.RedHat.RedHatFetchJob", "StellaOps.Feedser.Source.Distro.RedHat", TimeSpan.FromMinutes(12), TimeSpan.FromMinutes(6), "0,15,30,45 * * * *"), + new("source:redhat:parse", "StellaOps.Feedser.Source.Distro.RedHat.RedHatParseJob", "StellaOps.Feedser.Source.Distro.RedHat", TimeSpan.FromMinutes(15), TimeSpan.FromMinutes(6), "5,20,35,50 * * * *"), + new("source:redhat:map", "StellaOps.Feedser.Source.Distro.RedHat.RedHatMapJob", "StellaOps.Feedser.Source.Distro.RedHat", TimeSpan.FromMinutes(20), TimeSpan.FromMinutes(6), "10,25,40,55 * * * *"), + + new("source:cert-in:fetch", "StellaOps.Feedser.Source.CertIn.CertInFetchJob", "StellaOps.Feedser.Source.CertIn", TimeSpan.FromMinutes(15), TimeSpan.FromMinutes(5)), + new("source:cert-in:parse", "StellaOps.Feedser.Source.CertIn.CertInParseJob", "StellaOps.Feedser.Source.CertIn", TimeSpan.FromMinutes(15), TimeSpan.FromMinutes(5)), + new("source:cert-in:map", "StellaOps.Feedser.Source.CertIn.CertInMapJob", "StellaOps.Feedser.Source.CertIn", TimeSpan.FromMinutes(15), TimeSpan.FromMinutes(5)), + + new("source:cert-fr:fetch", "StellaOps.Feedser.Source.CertFr.CertFrFetchJob", "StellaOps.Feedser.Source.CertFr", TimeSpan.FromMinutes(15), TimeSpan.FromMinutes(5)), + new("source:cert-fr:parse", "StellaOps.Feedser.Source.CertFr.CertFrParseJob", "StellaOps.Feedser.Source.CertFr", TimeSpan.FromMinutes(15), TimeSpan.FromMinutes(5)), + new("source:cert-fr:map", "StellaOps.Feedser.Source.CertFr.CertFrMapJob", "StellaOps.Feedser.Source.CertFr", TimeSpan.FromMinutes(15), TimeSpan.FromMinutes(5)), + + new("source:jvn:fetch", "StellaOps.Feedser.Source.Jvn.JvnFetchJob", "StellaOps.Feedser.Source.Jvn", TimeSpan.FromMinutes(15), TimeSpan.FromMinutes(5)), + new("source:jvn:parse", "StellaOps.Feedser.Source.Jvn.JvnParseJob", "StellaOps.Feedser.Source.Jvn", TimeSpan.FromMinutes(15), TimeSpan.FromMinutes(5)), + new("source:jvn:map", "StellaOps.Feedser.Source.Jvn.JvnMapJob", "StellaOps.Feedser.Source.Jvn", TimeSpan.FromMinutes(15), TimeSpan.FromMinutes(5)), + + new("source:ics-kaspersky:fetch", "StellaOps.Feedser.Source.Ics.Kaspersky.KasperskyFetchJob", "StellaOps.Feedser.Source.Ics.Kaspersky", TimeSpan.FromMinutes(15), TimeSpan.FromMinutes(5)), + new("source:ics-kaspersky:parse", "StellaOps.Feedser.Source.Ics.Kaspersky.KasperskyParseJob", "StellaOps.Feedser.Source.Ics.Kaspersky", TimeSpan.FromMinutes(15), TimeSpan.FromMinutes(5)), + new("source:ics-kaspersky:map", "StellaOps.Feedser.Source.Ics.Kaspersky.KasperskyMapJob", "StellaOps.Feedser.Source.Ics.Kaspersky", TimeSpan.FromMinutes(15), TimeSpan.FromMinutes(5)), + + new("source:osv:fetch", "StellaOps.Feedser.Source.Osv.OsvFetchJob", "StellaOps.Feedser.Source.Osv", TimeSpan.FromMinutes(15), TimeSpan.FromMinutes(5)), + new("source:osv:parse", "StellaOps.Feedser.Source.Osv.OsvParseJob", "StellaOps.Feedser.Source.Osv", TimeSpan.FromMinutes(15), TimeSpan.FromMinutes(5)), + new("source:osv:map", "StellaOps.Feedser.Source.Osv.OsvMapJob", "StellaOps.Feedser.Source.Osv", TimeSpan.FromMinutes(15), TimeSpan.FromMinutes(5)), + + new("source:vmware:fetch", "StellaOps.Feedser.Source.Vndr.Vmware.VmwareFetchJob", "StellaOps.Feedser.Source.Vndr.Vmware", TimeSpan.FromMinutes(15), TimeSpan.FromMinutes(5)), + new("source:vmware:parse", "StellaOps.Feedser.Source.Vndr.Vmware.VmwareParseJob", "StellaOps.Feedser.Source.Vndr.Vmware", TimeSpan.FromMinutes(15), TimeSpan.FromMinutes(5)), + new("source:vmware:map", "StellaOps.Feedser.Source.Vndr.Vmware.VmwareMapJob", "StellaOps.Feedser.Source.Vndr.Vmware", TimeSpan.FromMinutes(15), TimeSpan.FromMinutes(5)), + + new("source:vndr-oracle:fetch", "StellaOps.Feedser.Source.Vndr.Oracle.OracleFetchJob", "StellaOps.Feedser.Source.Vndr.Oracle", TimeSpan.FromMinutes(15), TimeSpan.FromMinutes(5)), + new("source:vndr-oracle:parse", "StellaOps.Feedser.Source.Vndr.Oracle.OracleParseJob", "StellaOps.Feedser.Source.Vndr.Oracle", TimeSpan.FromMinutes(15), TimeSpan.FromMinutes(5)), + new("source:vndr-oracle:map", "StellaOps.Feedser.Source.Vndr.Oracle.OracleMapJob", "StellaOps.Feedser.Source.Vndr.Oracle", TimeSpan.FromMinutes(15), TimeSpan.FromMinutes(5)), + + new("export:json", "StellaOps.Feedser.Exporter.Json.JsonExportJob", "StellaOps.Feedser.Exporter.Json", TimeSpan.FromMinutes(10), TimeSpan.FromMinutes(5)), + new("export:trivy-db", "StellaOps.Feedser.Exporter.TrivyDb.TrivyDbExportJob", "StellaOps.Feedser.Exporter.TrivyDb", TimeSpan.FromMinutes(20), TimeSpan.FromMinutes(10)), + new("merge:reconcile", "StellaOps.Feedser.Merge.Jobs.MergeReconcileJob", "StellaOps.Feedser.Merge", TimeSpan.FromMinutes(15), TimeSpan.FromMinutes(5)) + }; + + public static IServiceCollection AddBuiltInFeedserJobs(this IServiceCollection services) + { + ArgumentNullException.ThrowIfNull(services); + + services.PostConfigure(options => + { + foreach (var registration in BuiltInJobs) + { + if (options.Definitions.ContainsKey(registration.Kind)) + { + continue; + } + + var jobType = Type.GetType( + $"{registration.JobType}, {registration.AssemblyName}", + throwOnError: false, + ignoreCase: false); + + if (jobType is null) + { + continue; + } + + var timeout = registration.Timeout > TimeSpan.Zero ? registration.Timeout : options.DefaultTimeout; + var lease = registration.LeaseDuration > TimeSpan.Zero ? registration.LeaseDuration : options.DefaultLeaseDuration; + + options.Definitions[registration.Kind] = new JobDefinition( + registration.Kind, + jobType, + timeout, + lease, + registration.CronExpression, + Enabled: true); + } + }); + + return services; + } +} diff --git a/src/StellaOps.Feedser.WebService/Extensions/TelemetryExtensions.cs b/src/StellaOps.Feedser.WebService/Extensions/TelemetryExtensions.cs new file mode 100644 index 00000000..bc9b54d5 --- /dev/null +++ b/src/StellaOps.Feedser.WebService/Extensions/TelemetryExtensions.cs @@ -0,0 +1,217 @@ +using System.Collections.Generic; +using System.Diagnostics; +using System.Linq; +using System.Reflection; +using Microsoft.AspNetCore.Builder; +using Microsoft.Extensions.DependencyInjection; +using OpenTelemetry.Metrics; +using OpenTelemetry.Resources; +using OpenTelemetry.Trace; +using Serilog; +using Serilog.Core; +using Serilog.Events; +using StellaOps.Feedser.Core.Jobs; +using StellaOps.Feedser.Source.Common.Telemetry; +using StellaOps.Feedser.WebService.Diagnostics; +using StellaOps.Feedser.WebService.Options; + +namespace StellaOps.Feedser.WebService.Extensions; + +public static class TelemetryExtensions +{ + public static void ConfigureFeedserTelemetry(this WebApplicationBuilder builder, FeedserOptions options) + { + ArgumentNullException.ThrowIfNull(builder); + ArgumentNullException.ThrowIfNull(options); + + var telemetry = options.Telemetry ?? new FeedserOptions.TelemetryOptions(); + + if (telemetry.EnableLogging) + { + builder.Host.UseSerilog((context, services, configuration) => + { + ConfigureSerilog(configuration, telemetry, builder.Environment.EnvironmentName, builder.Environment.ApplicationName); + }); + } + + if (!telemetry.Enabled || (!telemetry.EnableTracing && !telemetry.EnableMetrics)) + { + return; + } + + var openTelemetry = builder.Services.AddOpenTelemetry(); + + openTelemetry.ConfigureResource(resource => + { + var serviceName = telemetry.ServiceName ?? builder.Environment.ApplicationName; + var version = Assembly.GetExecutingAssembly().GetName().Version?.ToString() ?? "unknown"; + + resource.AddService(serviceName, serviceVersion: version, serviceInstanceId: Environment.MachineName); + resource.AddAttributes(new[] + { + new KeyValuePair("deployment.environment", builder.Environment.EnvironmentName), + }); + + foreach (var attribute in telemetry.ResourceAttributes) + { + if (string.IsNullOrWhiteSpace(attribute.Key) || attribute.Value is null) + { + continue; + } + + resource.AddAttributes(new[] { new KeyValuePair(attribute.Key, attribute.Value) }); + } + }); + + if (telemetry.EnableTracing) + { + openTelemetry.WithTracing(tracing => + { + tracing + .AddSource(JobDiagnostics.ActivitySourceName) + .AddSource(SourceDiagnostics.ActivitySourceName) + .AddAspNetCoreInstrumentation() + .AddHttpClientInstrumentation(); + + ConfigureExporters(telemetry, tracing); + }); + } + + if (telemetry.EnableMetrics) + { + openTelemetry.WithMetrics(metrics => + { + metrics + .AddMeter(JobDiagnostics.MeterName) + .AddMeter(SourceDiagnostics.MeterName) + .AddMeter("StellaOps.Feedser.Source.Nvd") + .AddMeter("StellaOps.Feedser.Source.Vndr.Chromium") + .AddMeter("StellaOps.Feedser.Source.Vndr.Adobe") + .AddMeter(JobMetrics.MeterName) + .AddAspNetCoreInstrumentation() + .AddHttpClientInstrumentation() + .AddRuntimeInstrumentation(); + + ConfigureExporters(telemetry, metrics); + }); + } + } + + private static void ConfigureSerilog(LoggerConfiguration configuration, FeedserOptions.TelemetryOptions telemetry, string environmentName, string applicationName) + { + if (!Enum.TryParse(telemetry.MinimumLogLevel, ignoreCase: true, out LogEventLevel level)) + { + level = LogEventLevel.Information; + } + + configuration + .MinimumLevel.Is(level) + .MinimumLevel.Override("Microsoft", LogEventLevel.Warning) + .MinimumLevel.Override("Microsoft.Hosting.Lifetime", LogEventLevel.Information) + .Enrich.FromLogContext() + .Enrich.With() + .Enrich.WithProperty("service.name", telemetry.ServiceName ?? applicationName) + .Enrich.WithProperty("deployment.environment", environmentName) + .WriteTo.Console(outputTemplate: "[{Timestamp:O}] [{Level:u3}] {Message:lj} {Properties}{NewLine}{Exception}"); + } + + private static void ConfigureExporters(FeedserOptions.TelemetryOptions telemetry, TracerProviderBuilder tracing) + { + if (string.IsNullOrWhiteSpace(telemetry.OtlpEndpoint)) + { + if (telemetry.ExportConsole) + { + tracing.AddConsoleExporter(); + } + + return; + } + + tracing.AddOtlpExporter(options => + { + options.Endpoint = new Uri(telemetry.OtlpEndpoint); + var headers = BuildHeaders(telemetry); + if (!string.IsNullOrEmpty(headers)) + { + options.Headers = headers; + } + }); + + if (telemetry.ExportConsole) + { + tracing.AddConsoleExporter(); + } + } + + private static void ConfigureExporters(FeedserOptions.TelemetryOptions telemetry, MeterProviderBuilder metrics) + { + if (string.IsNullOrWhiteSpace(telemetry.OtlpEndpoint)) + { + if (telemetry.ExportConsole) + { + metrics.AddConsoleExporter(); + } + + return; + } + + metrics.AddOtlpExporter(options => + { + options.Endpoint = new Uri(telemetry.OtlpEndpoint); + var headers = BuildHeaders(telemetry); + if (!string.IsNullOrEmpty(headers)) + { + options.Headers = headers; + } + }); + + if (telemetry.ExportConsole) + { + metrics.AddConsoleExporter(); + } + } + + private static string? BuildHeaders(FeedserOptions.TelemetryOptions telemetry) + { + if (telemetry.OtlpHeaders.Count == 0) + { + return null; + } + + return string.Join(",", telemetry.OtlpHeaders + .Where(static kvp => !string.IsNullOrWhiteSpace(kvp.Key) && !string.IsNullOrWhiteSpace(kvp.Value)) + .Select(static kvp => $"{kvp.Key}={kvp.Value}")); + } +} + +internal sealed class ActivityEnricher : ILogEventEnricher +{ + public void Enrich(LogEvent logEvent, ILogEventPropertyFactory propertyFactory) + { + var activity = Activity.Current; + if (activity is null) + { + return; + } + + if (activity.TraceId != default) + { + logEvent.AddPropertyIfAbsent(propertyFactory.CreateProperty("trace_id", activity.TraceId.ToString())); + } + + if (activity.SpanId != default) + { + logEvent.AddPropertyIfAbsent(propertyFactory.CreateProperty("span_id", activity.SpanId.ToString())); + } + + if (activity.ParentSpanId != default) + { + logEvent.AddPropertyIfAbsent(propertyFactory.CreateProperty("parent_span_id", activity.ParentSpanId.ToString())); + } + + if (!string.IsNullOrEmpty(activity.TraceStateString)) + { + logEvent.AddPropertyIfAbsent(propertyFactory.CreateProperty("trace_state", activity.TraceStateString)); + } + } +} diff --git a/src/StellaOps.Feedser.WebService/Filters/JobAuthorizationAuditFilter.cs b/src/StellaOps.Feedser.WebService/Filters/JobAuthorizationAuditFilter.cs new file mode 100644 index 00000000..3ef28d2b --- /dev/null +++ b/src/StellaOps.Feedser.WebService/Filters/JobAuthorizationAuditFilter.cs @@ -0,0 +1,104 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Net; +using System.Security.Claims; +using System.Threading.Tasks; +using Microsoft.AspNetCore.Http; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Options; +using StellaOps.Auth.Abstractions; +using StellaOps.Feedser.WebService.Options; + +namespace StellaOps.Feedser.WebService.Filters; + +/// +/// Emits structured audit logs for job endpoint authorization decisions, including bypass usage. +/// +public sealed class JobAuthorizationAuditFilter : IEndpointFilter +{ + internal const string LoggerName = "Feedser.Authorization.Audit"; + + public async ValueTask InvokeAsync(EndpointFilterInvocationContext context, EndpointFilterDelegate next) + { + ArgumentNullException.ThrowIfNull(context); + ArgumentNullException.ThrowIfNull(next); + + var httpContext = context.HttpContext; + var options = httpContext.RequestServices.GetRequiredService>().Value; + var authority = options.Authority; + + if (authority is null || !authority.Enabled) + { + return await next(context).ConfigureAwait(false); + } + + var logger = httpContext.RequestServices + .GetRequiredService() + .CreateLogger(LoggerName); + + var remoteAddress = httpContext.Connection.RemoteIpAddress; + var matcher = new NetworkMaskMatcher(authority.BypassNetworks); + var user = httpContext.User; + var isAuthenticated = user?.Identity?.IsAuthenticated ?? false; + var bypassUsed = !isAuthenticated && matcher.IsAllowed(remoteAddress); + + var result = await next(context).ConfigureAwait(false); + + var scopes = ExtractScopes(user); + var subject = user?.FindFirst(StellaOpsClaimTypes.Subject)?.Value; + var clientId = user?.FindFirst(StellaOpsClaimTypes.ClientId)?.Value; + + logger.LogInformation( + "Feedser authorization audit route={Route} status={StatusCode} subject={Subject} clientId={ClientId} scopes={Scopes} bypass={Bypass} remote={RemoteAddress}", + httpContext.Request.Path.Value ?? string.Empty, + httpContext.Response.StatusCode, + string.IsNullOrWhiteSpace(subject) ? "(anonymous)" : subject, + string.IsNullOrWhiteSpace(clientId) ? "(none)" : clientId, + scopes.Length == 0 ? "(none)" : string.Join(',', scopes), + bypassUsed, + remoteAddress?.ToString() ?? IPAddress.None.ToString()); + + return result; + } + + private static string[] ExtractScopes(ClaimsPrincipal? principal) + { + if (principal is null) + { + return Array.Empty(); + } + + var values = new HashSet(StringComparer.Ordinal); + + foreach (var claim in principal.FindAll(StellaOpsClaimTypes.ScopeItem)) + { + if (string.IsNullOrWhiteSpace(claim.Value)) + { + continue; + } + + values.Add(claim.Value); + } + + foreach (var claim in principal.FindAll(StellaOpsClaimTypes.Scope)) + { + if (string.IsNullOrWhiteSpace(claim.Value)) + { + continue; + } + + var parts = claim.Value.Split(' ', StringSplitOptions.RemoveEmptyEntries | StringSplitOptions.TrimEntries); + foreach (var part in parts) + { + var normalized = StellaOpsScopes.Normalize(part); + if (!string.IsNullOrEmpty(normalized)) + { + values.Add(normalized); + } + } + } + + return values.Count == 0 ? Array.Empty() : values.ToArray(); + } +} diff --git a/src/StellaOps.Feedser.WebService/Jobs/JobDefinitionResponse.cs b/src/StellaOps.Feedser.WebService/Jobs/JobDefinitionResponse.cs new file mode 100644 index 00000000..55101484 --- /dev/null +++ b/src/StellaOps.Feedser.WebService/Jobs/JobDefinitionResponse.cs @@ -0,0 +1,23 @@ +using StellaOps.Feedser.Core.Jobs; + +namespace StellaOps.Feedser.WebService.Jobs; + +public sealed record JobDefinitionResponse( + string Kind, + bool Enabled, + string? CronExpression, + TimeSpan Timeout, + TimeSpan LeaseDuration, + JobRunResponse? LastRun) +{ + public static JobDefinitionResponse FromDefinition(JobDefinition definition, JobRunSnapshot? lastRun) + { + return new JobDefinitionResponse( + definition.Kind, + definition.Enabled, + definition.CronExpression, + definition.Timeout, + definition.LeaseDuration, + lastRun is null ? null : JobRunResponse.FromSnapshot(lastRun)); + } +} diff --git a/src/StellaOps.Feedser.WebService/Jobs/JobRunResponse.cs b/src/StellaOps.Feedser.WebService/Jobs/JobRunResponse.cs new file mode 100644 index 00000000..9cb60ff0 --- /dev/null +++ b/src/StellaOps.Feedser.WebService/Jobs/JobRunResponse.cs @@ -0,0 +1,29 @@ +using StellaOps.Feedser.Core.Jobs; + +namespace StellaOps.Feedser.WebService.Jobs; + +public sealed record JobRunResponse( + Guid RunId, + string Kind, + JobRunStatus Status, + string Trigger, + DateTimeOffset CreatedAt, + DateTimeOffset? StartedAt, + DateTimeOffset? CompletedAt, + string? Error, + TimeSpan? Duration, + IReadOnlyDictionary Parameters) +{ + public static JobRunResponse FromSnapshot(JobRunSnapshot snapshot) + => new( + snapshot.RunId, + snapshot.Kind, + snapshot.Status, + snapshot.Trigger, + snapshot.CreatedAt, + snapshot.StartedAt, + snapshot.CompletedAt, + snapshot.Error, + snapshot.Duration, + snapshot.Parameters); +} diff --git a/src/StellaOps.Feedser.WebService/Jobs/JobTriggerRequest.cs b/src/StellaOps.Feedser.WebService/Jobs/JobTriggerRequest.cs new file mode 100644 index 00000000..18c1e443 --- /dev/null +++ b/src/StellaOps.Feedser.WebService/Jobs/JobTriggerRequest.cs @@ -0,0 +1,8 @@ +namespace StellaOps.Feedser.WebService.Jobs; + +public sealed class JobTriggerRequest +{ + public string Trigger { get; set; } = "api"; + + public Dictionary Parameters { get; set; } = new(StringComparer.Ordinal); +} diff --git a/src/StellaOps.Feedser.WebService/Options/FeedserOptions.cs b/src/StellaOps.Feedser.WebService/Options/FeedserOptions.cs new file mode 100644 index 00000000..d5df1bd7 --- /dev/null +++ b/src/StellaOps.Feedser.WebService/Options/FeedserOptions.cs @@ -0,0 +1,88 @@ +using System.Collections.Generic; + +namespace StellaOps.Feedser.WebService.Options; + +public sealed class FeedserOptions +{ + public StorageOptions Storage { get; set; } = new(); + + public PluginOptions Plugins { get; set; } = new(); + + public TelemetryOptions Telemetry { get; set; } = new(); + + public AuthorityOptions Authority { get; set; } = new(); + + public sealed class StorageOptions + { + public string Driver { get; set; } = "mongo"; + + public string Dsn { get; set; } = string.Empty; + + public string? Database { get; set; } + + public int CommandTimeoutSeconds { get; set; } = 30; + } + + public sealed class PluginOptions + { + public string? BaseDirectory { get; set; } + + public string? Directory { get; set; } + + public IList SearchPatterns { get; set; } = new List(); + } + + public sealed class TelemetryOptions + { + public bool Enabled { get; set; } = true; + + public bool EnableTracing { get; set; } = true; + + public bool EnableMetrics { get; set; } = true; + + public bool EnableLogging { get; set; } = true; + + public string MinimumLogLevel { get; set; } = "Information"; + + public string? ServiceName { get; set; } + + public string? OtlpEndpoint { get; set; } + + public IDictionary OtlpHeaders { get; set; } = new Dictionary(StringComparer.OrdinalIgnoreCase); + + public IDictionary ResourceAttributes { get; set; } = new Dictionary(StringComparer.OrdinalIgnoreCase); + + public bool ExportConsole { get; set; } + } + + public sealed class AuthorityOptions + { + public bool Enabled { get; set; } + + public bool AllowAnonymousFallback { get; set; } = true; + + public string Issuer { get; set; } = string.Empty; + + public string? MetadataAddress { get; set; } + + public bool RequireHttpsMetadata { get; set; } = true; + + public int BackchannelTimeoutSeconds { get; set; } = 30; + + public int TokenClockSkewSeconds { get; set; } = 60; + + public IList Audiences { get; set; } = new List(); + + public IList RequiredScopes { get; set; } = new List(); + + public IList BypassNetworks { get; set; } = new List(); + + public string? ClientId { get; set; } + + public string? ClientSecret { get; set; } + + public string? ClientSecretFile { get; set; } + + public IList ClientScopes { get; set; } = new List(); + } +} diff --git a/src/StellaOps.Feedser.WebService/Options/FeedserOptionsPostConfigure.cs b/src/StellaOps.Feedser.WebService/Options/FeedserOptionsPostConfigure.cs new file mode 100644 index 00000000..6de16e6e --- /dev/null +++ b/src/StellaOps.Feedser.WebService/Options/FeedserOptionsPostConfigure.cs @@ -0,0 +1,46 @@ +using System; +using System.IO; + +namespace StellaOps.Feedser.WebService.Options; + +/// +/// Post-configuration helpers for . +/// +public static class FeedserOptionsPostConfigure +{ + /// + /// Applies derived settings that require filesystem access, such as loading client secrets from disk. + /// + /// The options to mutate. + /// Application content root used to resolve relative paths. + public static void Apply(FeedserOptions options, string contentRootPath) + { + ArgumentNullException.ThrowIfNull(options); + + options.Authority ??= new FeedserOptions.AuthorityOptions(); + + var authority = options.Authority; + if (string.IsNullOrWhiteSpace(authority.ClientSecret) + && !string.IsNullOrWhiteSpace(authority.ClientSecretFile)) + { + var resolvedPath = authority.ClientSecretFile!; + if (!Path.IsPathRooted(resolvedPath)) + { + resolvedPath = Path.Combine(contentRootPath, resolvedPath); + } + + if (!File.Exists(resolvedPath)) + { + throw new InvalidOperationException($"Authority client secret file '{resolvedPath}' was not found."); + } + + var secret = File.ReadAllText(resolvedPath).Trim(); + if (string.IsNullOrEmpty(secret)) + { + throw new InvalidOperationException($"Authority client secret file '{resolvedPath}' is empty."); + } + + authority.ClientSecret = secret; + } + } +} diff --git a/src/StellaOps.Feedser.WebService/Options/FeedserOptionsValidator.cs b/src/StellaOps.Feedser.WebService/Options/FeedserOptionsValidator.cs new file mode 100644 index 00000000..9d6e4883 --- /dev/null +++ b/src/StellaOps.Feedser.WebService/Options/FeedserOptionsValidator.cs @@ -0,0 +1,166 @@ +using System; +using System.Collections.Generic; +using Microsoft.Extensions.Logging; +using StellaOps.Auth.Abstractions; + +namespace StellaOps.Feedser.WebService.Options; + +public static class FeedserOptionsValidator +{ + public static void Validate(FeedserOptions options) + { + ArgumentNullException.ThrowIfNull(options); + + if (!string.Equals(options.Storage.Driver, "mongo", StringComparison.OrdinalIgnoreCase)) + { + throw new InvalidOperationException("Only Mongo storage driver is supported (storage.driver == 'mongo')."); + } + + if (string.IsNullOrWhiteSpace(options.Storage.Dsn)) + { + throw new InvalidOperationException("Storage DSN must be configured."); + } + + if (options.Storage.CommandTimeoutSeconds <= 0) + { + throw new InvalidOperationException("Command timeout must be greater than zero seconds."); + } + + options.Telemetry ??= new FeedserOptions.TelemetryOptions(); + + options.Authority ??= new FeedserOptions.AuthorityOptions(); + NormalizeList(options.Authority.Audiences, toLower: false); + NormalizeList(options.Authority.RequiredScopes, toLower: true); + NormalizeList(options.Authority.BypassNetworks, toLower: false); + NormalizeList(options.Authority.ClientScopes, toLower: true); + + if (options.Authority.RequiredScopes.Count == 0) + { + options.Authority.RequiredScopes.Add(StellaOpsScopes.FeedserJobsTrigger); + } + + if (options.Authority.ClientScopes.Count == 0) + { + foreach (var scope in options.Authority.RequiredScopes) + { + options.Authority.ClientScopes.Add(scope); + } + } + + if (options.Authority.ClientScopes.Count == 0) + { + options.Authority.ClientScopes.Add(StellaOpsScopes.FeedserJobsTrigger); + } + + if (options.Authority.BackchannelTimeoutSeconds <= 0) + { + throw new InvalidOperationException("Authority backchannelTimeoutSeconds must be greater than zero."); + } + + if (options.Authority.TokenClockSkewSeconds < 0 || options.Authority.TokenClockSkewSeconds > 300) + { + throw new InvalidOperationException("Authority tokenClockSkewSeconds must be between 0 and 300 seconds."); + } + + if (options.Authority.Enabled) + { + if (string.IsNullOrWhiteSpace(options.Authority.Issuer)) + { + throw new InvalidOperationException("Authority issuer must be configured when authority is enabled."); + } + + if (!Uri.TryCreate(options.Authority.Issuer, UriKind.Absolute, out var issuerUri)) + { + throw new InvalidOperationException("Authority issuer must be an absolute URI."); + } + + if (options.Authority.RequireHttpsMetadata && !issuerUri.IsLoopback && !string.Equals(issuerUri.Scheme, Uri.UriSchemeHttps, StringComparison.OrdinalIgnoreCase)) + { + throw new InvalidOperationException("Authority issuer must use HTTPS when requireHttpsMetadata is enabled."); + } + + if (!string.IsNullOrWhiteSpace(options.Authority.MetadataAddress) && !Uri.TryCreate(options.Authority.MetadataAddress, UriKind.Absolute, out _)) + { + throw new InvalidOperationException("Authority metadataAddress must be an absolute URI when specified."); + } + + if (options.Authority.Audiences.Count == 0) + { + throw new InvalidOperationException("Authority audiences must include at least one entry when authority is enabled."); + } + + if (!options.Authority.AllowAnonymousFallback) + { + if (string.IsNullOrWhiteSpace(options.Authority.ClientId)) + { + throw new InvalidOperationException("Authority clientId must be configured when anonymous fallback is disabled."); + } + + if (string.IsNullOrWhiteSpace(options.Authority.ClientSecret)) + { + throw new InvalidOperationException("Authority clientSecret must be configured when anonymous fallback is disabled."); + } + } + } + + if (!Enum.TryParse(options.Telemetry.MinimumLogLevel, ignoreCase: true, out LogLevel _)) + { + throw new InvalidOperationException($"Telemetry minimum log level '{options.Telemetry.MinimumLogLevel}' is invalid."); + } + + if (!string.IsNullOrWhiteSpace(options.Telemetry.OtlpEndpoint) && !Uri.TryCreate(options.Telemetry.OtlpEndpoint, UriKind.Absolute, out _)) + { + throw new InvalidOperationException("Telemetry OTLP endpoint must be an absolute URI."); + } + + foreach (var attribute in options.Telemetry.ResourceAttributes) + { + if (string.IsNullOrWhiteSpace(attribute.Key)) + { + throw new InvalidOperationException("Telemetry resource attribute keys must be non-empty."); + } + } + + foreach (var header in options.Telemetry.OtlpHeaders) + { + if (string.IsNullOrWhiteSpace(header.Key)) + { + throw new InvalidOperationException("Telemetry OTLP header names must be non-empty."); + } + } + } + + private static void NormalizeList(IList values, bool toLower) + { + if (values is null || values.Count == 0) + { + return; + } + + var seen = new HashSet(StringComparer.OrdinalIgnoreCase); + + for (var index = values.Count - 1; index >= 0; index--) + { + var entry = values[index]; + if (string.IsNullOrWhiteSpace(entry)) + { + values.RemoveAt(index); + continue; + } + + var normalized = entry.Trim(); + if (toLower) + { + normalized = normalized.ToLowerInvariant(); + } + + if (!seen.Add(normalized)) + { + values.RemoveAt(index); + continue; + } + + values[index] = normalized; + } + } +} diff --git a/src/StellaOps.Feedser.WebService/Program.cs b/src/StellaOps.Feedser.WebService/Program.cs new file mode 100644 index 00000000..094dd0d0 --- /dev/null +++ b/src/StellaOps.Feedser.WebService/Program.cs @@ -0,0 +1,672 @@ +using System.Collections.Generic; +using System.Linq; +using System.Text; +using Microsoft.AspNetCore.Diagnostics; +using Microsoft.AspNetCore.Http; +using Microsoft.AspNetCore.Mvc; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Hosting; +using System.Diagnostics; +using System.Text.Json; +using System.Text.Json.Serialization; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Options; +using MongoDB.Bson; +using MongoDB.Driver; +using StellaOps.Feedser.Core.Jobs; +using StellaOps.Feedser.Storage.Mongo; +using StellaOps.Feedser.WebService.Diagnostics; +using Serilog; +using StellaOps.Feedser.Merge; +using StellaOps.Feedser.Merge.Services; +using StellaOps.Feedser.WebService.Extensions; +using StellaOps.Feedser.WebService.Jobs; +using StellaOps.Feedser.WebService.Options; +using StellaOps.Feedser.WebService.Filters; +using Serilog.Events; +using StellaOps.Plugin.DependencyInjection; +using StellaOps.Plugin.Hosting; +using StellaOps.Configuration; +using StellaOps.Auth.Abstractions; +using StellaOps.Auth.ServerIntegration; + +var builder = WebApplication.CreateBuilder(args); + +const string JobsPolicyName = "Feedser.Jobs.Trigger"; + +builder.Configuration.AddStellaOpsDefaults(options => +{ + options.BasePath = builder.Environment.ContentRootPath; + options.EnvironmentPrefix = "FEEDSER_"; + options.ConfigureBuilder = configurationBuilder => + { + configurationBuilder.AddFeedserYaml(Path.Combine(builder.Environment.ContentRootPath, "../etc/feedser.yaml")); + }; +}); + +var contentRootPath = builder.Environment.ContentRootPath; + +var feedserOptions = builder.Configuration.BindOptions(postConfigure: (opts, _) => +{ + FeedserOptionsPostConfigure.Apply(opts, contentRootPath); + FeedserOptionsValidator.Validate(opts); +}); +builder.Services.AddOptions() + .Bind(builder.Configuration) + .PostConfigure(options => + { + FeedserOptionsPostConfigure.Apply(options, contentRootPath); + FeedserOptionsValidator.Validate(options); + }) + .ValidateOnStart(); + +builder.ConfigureFeedserTelemetry(feedserOptions); + +builder.Services.AddMongoStorage(storageOptions => +{ + storageOptions.ConnectionString = feedserOptions.Storage.Dsn; + storageOptions.DatabaseName = feedserOptions.Storage.Database; + storageOptions.CommandTimeout = TimeSpan.FromSeconds(feedserOptions.Storage.CommandTimeoutSeconds); +}); + +builder.Services.AddMergeModule(builder.Configuration); +builder.Services.AddJobScheduler(); +builder.Services.AddBuiltInFeedserJobs(); + +builder.Services.AddSingleton(sp => new ServiceStatus(sp.GetRequiredService())); + +var authorityConfigured = feedserOptions.Authority is { Enabled: true }; + +if (authorityConfigured) +{ + builder.Services.AddStellaOpsResourceServerAuthentication( + builder.Configuration, + configurationSection: null, + configure: resourceOptions => + { + resourceOptions.Authority = feedserOptions.Authority.Issuer; + resourceOptions.RequireHttpsMetadata = feedserOptions.Authority.RequireHttpsMetadata; + resourceOptions.BackchannelTimeout = TimeSpan.FromSeconds(feedserOptions.Authority.BackchannelTimeoutSeconds); + resourceOptions.TokenClockSkew = TimeSpan.FromSeconds(feedserOptions.Authority.TokenClockSkewSeconds); + + if (!string.IsNullOrWhiteSpace(feedserOptions.Authority.MetadataAddress)) + { + resourceOptions.MetadataAddress = feedserOptions.Authority.MetadataAddress; + } + + foreach (var audience in feedserOptions.Authority.Audiences) + { + resourceOptions.Audiences.Add(audience); + } + + foreach (var scope in feedserOptions.Authority.RequiredScopes) + { + resourceOptions.RequiredScopes.Add(scope); + } + + foreach (var network in feedserOptions.Authority.BypassNetworks) + { + resourceOptions.BypassNetworks.Add(network); + } + }); + + builder.Services.AddAuthorization(options => + { + options.AddStellaOpsScopePolicy(JobsPolicyName, feedserOptions.Authority.RequiredScopes.ToArray()); + }); +} + +var pluginHostOptions = BuildPluginOptions(feedserOptions, builder.Environment.ContentRootPath); +builder.Services.RegisterPluginRoutines(builder.Configuration, pluginHostOptions); + +builder.Services.AddEndpointsApiExplorer(); + +var app = builder.Build(); + +var resolvedFeedserOptions = app.Services.GetRequiredService>().Value; +var resolvedAuthority = resolvedFeedserOptions.Authority ?? new FeedserOptions.AuthorityOptions(); +authorityConfigured = resolvedAuthority.Enabled; +var enforceAuthority = resolvedAuthority.Enabled && !resolvedAuthority.AllowAnonymousFallback; + +if (resolvedAuthority.Enabled && resolvedAuthority.AllowAnonymousFallback) +{ + app.Logger.LogWarning( + "Authority authentication is configured but anonymous fallback remains enabled. Set authority.allowAnonymousFallback to false before 2025-12-31 to complete the rollout."); +} + +var jsonOptions = new JsonSerializerOptions(JsonSerializerDefaults.Web); +jsonOptions.Converters.Add(new JsonStringEnumConverter()); + +var loggingEnabled = feedserOptions.Telemetry?.EnableLogging ?? true; + +if (loggingEnabled) +{ + app.UseSerilogRequestLogging(options => + { + options.IncludeQueryInRequestPath = true; + options.GetLevel = (httpContext, elapsedMs, exception) => exception is null ? LogEventLevel.Information : LogEventLevel.Error; + options.EnrichDiagnosticContext = (diagnosticContext, httpContext) => + { + diagnosticContext.Set("RequestId", httpContext.TraceIdentifier); + diagnosticContext.Set("UserAgent", httpContext.Request.Headers.UserAgent.ToString()); + if (Activity.Current is { TraceId: var traceId } && traceId != default) + { + diagnosticContext.Set("TraceId", traceId.ToString()); + } + }; + }); +} + +app.UseExceptionHandler(errorApp => +{ + errorApp.Run(async context => + { + context.Response.ContentType = "application/problem+json"; + var feature = context.Features.Get(); + var error = feature?.Error; + + var extensions = new Dictionary(StringComparer.Ordinal) + { + ["traceId"] = Activity.Current?.TraceId.ToString() ?? context.TraceIdentifier, + }; + + var problem = Results.Problem( + detail: error?.Message, + instance: context.Request.Path, + statusCode: StatusCodes.Status500InternalServerError, + title: "Unexpected server error", + type: ProblemTypes.JobFailure, + extensions: extensions); + + await problem.ExecuteAsync(context); + }); +}); + +if (authorityConfigured) +{ + app.Use(async (context, next) => + { + await next().ConfigureAwait(false); + + if (!context.Request.Path.StartsWithSegments("/jobs", StringComparison.OrdinalIgnoreCase)) + { + return; + } + + if (context.Response.StatusCode != StatusCodes.Status401Unauthorized) + { + return; + } + + var optionsMonitor = context.RequestServices.GetRequiredService>().Value.Authority; + if (optionsMonitor is null || !optionsMonitor.Enabled) + { + return; + } + + var logger = context.RequestServices + .GetRequiredService() + .CreateLogger(JobAuthorizationAuditFilter.LoggerName); + + var matcher = new NetworkMaskMatcher(optionsMonitor.BypassNetworks); + var remote = context.Connection.RemoteIpAddress; + var bypassAllowed = matcher.IsAllowed(remote); + + logger.LogWarning( + "Feedser authorization denied route={Route} remote={RemoteAddress} bypassAllowed={BypassAllowed} hasPrincipal={HasPrincipal}", + context.Request.Path.Value ?? string.Empty, + remote?.ToString() ?? "unknown", + bypassAllowed, + context.User?.Identity?.IsAuthenticated ?? false); + }); +} + +if (authorityConfigured) +{ + app.UseAuthentication(); + app.UseAuthorization(); +} + +IResult JsonResult(T value, int? statusCode = null) +{ + var payload = JsonSerializer.Serialize(value, jsonOptions); + return Results.Content(payload, "application/json", Encoding.UTF8, statusCode); +} + +IResult Problem(HttpContext context, string title, int statusCode, string type, string? detail = null, IDictionary? extensions = null) +{ + var traceId = Activity.Current?.TraceId.ToString() ?? context.TraceIdentifier; + extensions ??= new Dictionary(StringComparer.Ordinal) + { + ["traceId"] = traceId, + }; + + if (!extensions.ContainsKey("traceId")) + { + extensions["traceId"] = traceId; + } + + var problemDetails = new ProblemDetails + { + Type = type, + Title = title, + Detail = detail, + Status = statusCode, + Instance = context.Request.Path + }; + + foreach (var entry in extensions) + { + problemDetails.Extensions[entry.Key] = entry.Value; + } + + var payload = JsonSerializer.Serialize(problemDetails, jsonOptions); + return Results.Content(payload, "application/problem+json", Encoding.UTF8, statusCode); +} + +static KeyValuePair[] BuildJobMetricTags(string jobKind, string trigger, string outcome) + => new[] + { + new KeyValuePair("job.kind", jobKind), + new KeyValuePair("job.trigger", trigger), + new KeyValuePair("job.outcome", outcome), + }; + +void ApplyNoCache(HttpResponse response) +{ + if (response is null) + { + return; + } + + response.Headers.CacheControl = "no-store, no-cache, max-age=0, must-revalidate"; + response.Headers.Pragma = "no-cache"; + response.Headers["Expires"] = "0"; +} + +await InitializeMongoAsync(app); + +app.MapGet("/health", (IOptions opts, ServiceStatus status, HttpContext context) => +{ + ApplyNoCache(context.Response); + + var snapshot = status.CreateSnapshot(); + var uptimeSeconds = Math.Max((snapshot.CapturedAt - snapshot.StartedAt).TotalSeconds, 0d); + + var storage = new StorageBootstrapHealth( + Driver: opts.Value.Storage.Driver, + Completed: snapshot.BootstrapCompletedAt is not null, + CompletedAt: snapshot.BootstrapCompletedAt, + DurationMs: snapshot.BootstrapDuration?.TotalMilliseconds); + + var telemetry = new TelemetryHealth( + Enabled: opts.Value.Telemetry.Enabled, + Tracing: opts.Value.Telemetry.EnableTracing, + Metrics: opts.Value.Telemetry.EnableMetrics, + Logging: opts.Value.Telemetry.EnableLogging); + + var response = new HealthDocument( + Status: "healthy", + StartedAt: snapshot.StartedAt, + UptimeSeconds: uptimeSeconds, + Storage: storage, + Telemetry: telemetry); + + return JsonResult(response); +}); + +app.MapGet("/ready", async (IMongoDatabase database, ServiceStatus status, HttpContext context, CancellationToken cancellationToken) => +{ + ApplyNoCache(context.Response); + + var stopwatch = Stopwatch.StartNew(); + try + { + await database.RunCommandAsync((Command)"{ ping: 1 }", cancellationToken: cancellationToken).ConfigureAwait(false); + stopwatch.Stop(); + status.RecordMongoCheck(success: true, latency: stopwatch.Elapsed, error: null); + + var snapshot = status.CreateSnapshot(); + var uptimeSeconds = Math.Max((snapshot.CapturedAt - snapshot.StartedAt).TotalSeconds, 0d); + + var mongo = new MongoReadyHealth( + Status: "ready", + LatencyMs: snapshot.LastMongoLatency?.TotalMilliseconds, + CheckedAt: snapshot.LastReadyCheckAt, + Error: null); + + var response = new ReadyDocument( + Status: "ready", + StartedAt: snapshot.StartedAt, + UptimeSeconds: uptimeSeconds, + Mongo: mongo); + + return JsonResult(response); + } + catch (Exception ex) + { + stopwatch.Stop(); + status.RecordMongoCheck(success: false, latency: stopwatch.Elapsed, error: ex.Message); + + var snapshot = status.CreateSnapshot(); + var uptimeSeconds = Math.Max((snapshot.CapturedAt - snapshot.StartedAt).TotalSeconds, 0d); + + var mongo = new MongoReadyHealth( + Status: "unready", + LatencyMs: snapshot.LastMongoLatency?.TotalMilliseconds, + CheckedAt: snapshot.LastReadyCheckAt, + Error: snapshot.LastMongoError ?? ex.Message); + + var response = new ReadyDocument( + Status: "unready", + StartedAt: snapshot.StartedAt, + UptimeSeconds: uptimeSeconds, + Mongo: mongo); + + var extensions = new Dictionary(StringComparer.Ordinal) + { + ["mongoLatencyMs"] = snapshot.LastMongoLatency?.TotalMilliseconds, + ["mongoError"] = snapshot.LastMongoError ?? ex.Message, + }; + + return Problem(context, "Mongo unavailable", StatusCodes.Status503ServiceUnavailable, ProblemTypes.ServiceUnavailable, snapshot.LastMongoError ?? ex.Message, extensions); + } +}); + +app.MapGet("/diagnostics/aliases/{seed}", async (string seed, AliasGraphResolver resolver, HttpContext context, CancellationToken cancellationToken) => +{ + ApplyNoCache(context.Response); + + if (string.IsNullOrWhiteSpace(seed)) + { + return Problem(context, "Seed advisory key is required.", StatusCodes.Status400BadRequest, ProblemTypes.Validation); + } + + var component = await resolver.BuildComponentAsync(seed, cancellationToken).ConfigureAwait(false); + + var aliases = component.AliasMap.ToDictionary( + static kvp => kvp.Key, + static kvp => kvp.Value + .Select(record => new + { + record.Scheme, + record.Value, + UpdatedAt = record.UpdatedAt + }) + .ToArray()); + + var response = new + { + Seed = component.SeedAdvisoryKey, + Advisories = component.AdvisoryKeys, + Collisions = component.Collisions + .Select(collision => new + { + collision.Scheme, + collision.Value, + AdvisoryKeys = collision.AdvisoryKeys + }) + .ToArray(), + Aliases = aliases + }; + + return JsonResult(response); +}); + +var jobsListEndpoint = app.MapGet("/jobs", async (string? kind, int? limit, IJobCoordinator coordinator, HttpContext context, CancellationToken cancellationToken) => +{ + ApplyNoCache(context.Response); + + var take = Math.Clamp(limit.GetValueOrDefault(50), 1, 200); + var runs = await coordinator.GetRecentRunsAsync(kind, take, cancellationToken).ConfigureAwait(false); + var payload = runs.Select(JobRunResponse.FromSnapshot).ToArray(); + return JsonResult(payload); +}).AddEndpointFilter(); +if (enforceAuthority) +{ + jobsListEndpoint.RequireAuthorization(JobsPolicyName); +} + +var jobByIdEndpoint = app.MapGet("/jobs/{runId:guid}", async (Guid runId, IJobCoordinator coordinator, HttpContext context, CancellationToken cancellationToken) => +{ + ApplyNoCache(context.Response); + + var run = await coordinator.GetRunAsync(runId, cancellationToken).ConfigureAwait(false); + if (run is null) + { + return Problem(context, "Job run not found", StatusCodes.Status404NotFound, ProblemTypes.NotFound, $"Job run '{runId}' was not found."); + } + + return JsonResult(JobRunResponse.FromSnapshot(run)); +}).AddEndpointFilter(); +if (enforceAuthority) +{ + jobByIdEndpoint.RequireAuthorization(JobsPolicyName); +} + +var jobDefinitionsEndpoint = app.MapGet("/jobs/definitions", async (IJobCoordinator coordinator, HttpContext context, CancellationToken cancellationToken) => +{ + ApplyNoCache(context.Response); + + var definitions = await coordinator.GetDefinitionsAsync(cancellationToken).ConfigureAwait(false); + if (definitions.Count == 0) + { + return JsonResult(Array.Empty()); + } + + var definitionKinds = definitions.Select(static definition => definition.Kind).ToArray(); + var lastRuns = await coordinator.GetLastRunsAsync(definitionKinds, cancellationToken).ConfigureAwait(false); + + var responses = new List(definitions.Count); + foreach (var definition in definitions) + { + lastRuns.TryGetValue(definition.Kind, out var lastRun); + responses.Add(JobDefinitionResponse.FromDefinition(definition, lastRun)); + } + + return JsonResult(responses); +}).AddEndpointFilter(); +if (enforceAuthority) +{ + jobDefinitionsEndpoint.RequireAuthorization(JobsPolicyName); +} + +var jobDefinitionEndpoint = app.MapGet("/jobs/definitions/{kind}", async (string kind, IJobCoordinator coordinator, HttpContext context, CancellationToken cancellationToken) => +{ + ApplyNoCache(context.Response); + + var definition = (await coordinator.GetDefinitionsAsync(cancellationToken).ConfigureAwait(false)) + .FirstOrDefault(d => string.Equals(d.Kind, kind, StringComparison.Ordinal)); + + if (definition is null) + { + return Problem(context, "Job definition not found", StatusCodes.Status404NotFound, ProblemTypes.NotFound, $"Job kind '{kind}' is not registered."); + } + + var lastRuns = await coordinator.GetLastRunsAsync(new[] { definition.Kind }, cancellationToken).ConfigureAwait(false); + lastRuns.TryGetValue(definition.Kind, out var lastRun); + + var response = JobDefinitionResponse.FromDefinition(definition, lastRun); + return JsonResult(response); +}).AddEndpointFilter(); +if (enforceAuthority) +{ + jobDefinitionEndpoint.RequireAuthorization(JobsPolicyName); +} + +var jobDefinitionRunsEndpoint = app.MapGet("/jobs/definitions/{kind}/runs", async (string kind, int? limit, IJobCoordinator coordinator, HttpContext context, CancellationToken cancellationToken) => +{ + ApplyNoCache(context.Response); + + var definition = (await coordinator.GetDefinitionsAsync(cancellationToken).ConfigureAwait(false)) + .FirstOrDefault(d => string.Equals(d.Kind, kind, StringComparison.Ordinal)); + + if (definition is null) + { + return Problem(context, "Job definition not found", StatusCodes.Status404NotFound, ProblemTypes.NotFound, $"Job kind '{kind}' is not registered."); + } + + var take = Math.Clamp(limit.GetValueOrDefault(20), 1, 200); + var runs = await coordinator.GetRecentRunsAsync(kind, take, cancellationToken).ConfigureAwait(false); + var payload = runs.Select(JobRunResponse.FromSnapshot).ToArray(); + return JsonResult(payload); +}).AddEndpointFilter(); +if (enforceAuthority) +{ + jobDefinitionRunsEndpoint.RequireAuthorization(JobsPolicyName); +} + +var activeJobsEndpoint = app.MapGet("/jobs/active", async (IJobCoordinator coordinator, HttpContext context, CancellationToken cancellationToken) => +{ + ApplyNoCache(context.Response); + + var runs = await coordinator.GetActiveRunsAsync(cancellationToken).ConfigureAwait(false); + var payload = runs.Select(JobRunResponse.FromSnapshot).ToArray(); + return JsonResult(payload); +}).AddEndpointFilter(); +if (enforceAuthority) +{ + activeJobsEndpoint.RequireAuthorization(JobsPolicyName); +} + +var triggerJobEndpoint = app.MapPost("/jobs/{*jobKind}", async (string jobKind, JobTriggerRequest request, IJobCoordinator coordinator, HttpContext context) => +{ + ApplyNoCache(context.Response); + + request ??= new JobTriggerRequest(); + request.Parameters ??= new Dictionary(StringComparer.Ordinal); + var trigger = string.IsNullOrWhiteSpace(request.Trigger) ? "api" : request.Trigger; + + var lifetime = context.RequestServices.GetRequiredService(); + var result = await coordinator.TriggerAsync(jobKind, request.Parameters, trigger, lifetime.ApplicationStopping).ConfigureAwait(false); + + var outcome = result.Outcome; + var tags = BuildJobMetricTags(jobKind, trigger, outcome.ToString().ToLowerInvariant()); + + switch (outcome) + { + case JobTriggerOutcome.Accepted: + JobMetrics.TriggerCounter.Add(1, tags); + if (result.Run is null) + { + return Results.StatusCode(StatusCodes.Status202Accepted); + } + + var acceptedRun = JobRunResponse.FromSnapshot(result.Run); + context.Response.Headers.Location = $"/jobs/{acceptedRun.RunId}"; + return JsonResult(acceptedRun, StatusCodes.Status202Accepted); + + case JobTriggerOutcome.NotFound: + JobMetrics.TriggerConflictCounter.Add(1, tags); + return Problem(context, "Job not found", StatusCodes.Status404NotFound, ProblemTypes.NotFound, result.ErrorMessage ?? $"Job '{jobKind}' is not registered."); + + case JobTriggerOutcome.Disabled: + JobMetrics.TriggerConflictCounter.Add(1, tags); + return Problem(context, "Job disabled", StatusCodes.Status423Locked, ProblemTypes.Locked, result.ErrorMessage ?? $"Job '{jobKind}' is disabled."); + + case JobTriggerOutcome.AlreadyRunning: + JobMetrics.TriggerConflictCounter.Add(1, tags); + return Problem(context, "Job already running", StatusCodes.Status409Conflict, ProblemTypes.Conflict, result.ErrorMessage ?? $"Job '{jobKind}' already has an active run."); + + case JobTriggerOutcome.LeaseRejected: + JobMetrics.TriggerConflictCounter.Add(1, tags); + return Problem(context, "Job lease rejected", StatusCodes.Status409Conflict, ProblemTypes.LeaseRejected, result.ErrorMessage ?? $"Job '{jobKind}' could not acquire a lease."); + + case JobTriggerOutcome.InvalidParameters: + { + JobMetrics.TriggerConflictCounter.Add(1, tags); + var extensions = new Dictionary(StringComparer.Ordinal) + { + ["parameters"] = request.Parameters, + }; + return Problem(context, "Invalid job parameters", StatusCodes.Status400BadRequest, ProblemTypes.Validation, result.ErrorMessage, extensions); + } + + case JobTriggerOutcome.Cancelled: + { + JobMetrics.TriggerConflictCounter.Add(1, tags); + var extensions = new Dictionary(StringComparer.Ordinal) + { + ["run"] = result.Run is null ? null : JobRunResponse.FromSnapshot(result.Run), + }; + + return Problem(context, "Job cancelled", StatusCodes.Status409Conflict, ProblemTypes.Conflict, result.ErrorMessage ?? $"Job '{jobKind}' was cancelled before completion.", extensions); + } + + case JobTriggerOutcome.Failed: + { + JobMetrics.TriggerFailureCounter.Add(1, tags); + var extensions = new Dictionary(StringComparer.Ordinal) + { + ["run"] = result.Run is null ? null : JobRunResponse.FromSnapshot(result.Run), + }; + + return Problem(context, "Job execution failed", StatusCodes.Status500InternalServerError, ProblemTypes.JobFailure, result.ErrorMessage, extensions); + } + + default: + JobMetrics.TriggerFailureCounter.Add(1, tags); + return Problem(context, "Unexpected job outcome", StatusCodes.Status500InternalServerError, ProblemTypes.JobFailure, $"Job '{jobKind}' returned outcome '{outcome}'."); + } +}).AddEndpointFilter(); +if (enforceAuthority) +{ + triggerJobEndpoint.RequireAuthorization(JobsPolicyName); +} + +await app.RunAsync(); + +static PluginHostOptions BuildPluginOptions(FeedserOptions options, string contentRoot) +{ + var pluginOptions = new PluginHostOptions + { + BaseDirectory = options.Plugins.BaseDirectory ?? contentRoot, + PluginsDirectory = options.Plugins.Directory ?? Path.Combine(contentRoot, "PluginBinaries"), + EnsureDirectoryExists = true, + RecursiveSearch = false, + }; + + if (options.Plugins.SearchPatterns.Count == 0) + { + pluginOptions.SearchPatterns.Add("StellaOps.Feedser.Plugin.*.dll"); + } + else + { + foreach (var pattern in options.Plugins.SearchPatterns) + { + if (!string.IsNullOrWhiteSpace(pattern)) + { + pluginOptions.SearchPatterns.Add(pattern); + } + } + } + + return pluginOptions; +} + +static async Task InitializeMongoAsync(WebApplication app) +{ + await using var scope = app.Services.CreateAsyncScope(); + var bootstrapper = scope.ServiceProvider.GetRequiredService(); + var logger = scope.ServiceProvider.GetRequiredService().CreateLogger("MongoBootstrapper"); + var status = scope.ServiceProvider.GetRequiredService(); + + var stopwatch = Stopwatch.StartNew(); + + try + { + await bootstrapper.InitializeAsync(app.Lifetime.ApplicationStopping).ConfigureAwait(false); + stopwatch.Stop(); + status.MarkBootstrapCompleted(stopwatch.Elapsed); + logger.LogInformation("Mongo bootstrap completed in {ElapsedMs} ms", stopwatch.Elapsed.TotalMilliseconds); + } + catch (Exception ex) + { + stopwatch.Stop(); + status.RecordMongoCheck(success: false, latency: stopwatch.Elapsed, error: ex.Message); + logger.LogCritical(ex, "Mongo bootstrap failed after {ElapsedMs} ms", stopwatch.Elapsed.TotalMilliseconds); + throw; + } +} + +public partial class Program; diff --git a/src/StellaOps.Feedser.WebService/Properties/launchSettings.json b/src/StellaOps.Feedser.WebService/Properties/launchSettings.json new file mode 100644 index 00000000..261bc6f8 --- /dev/null +++ b/src/StellaOps.Feedser.WebService/Properties/launchSettings.json @@ -0,0 +1,12 @@ +{ + "profiles": { + "StellaOps.Feedser.WebService": { + "commandName": "Project", + "launchBrowser": true, + "environmentVariables": { + "ASPNETCORE_ENVIRONMENT": "Development" + }, + "applicationUrl": "https://localhost:50411;http://localhost:50412" + } + } +} \ No newline at end of file diff --git a/src/StellaOps.Feedser.WebService/StellaOps.Feedser.WebService.csproj b/src/StellaOps.Feedser.WebService/StellaOps.Feedser.WebService.csproj new file mode 100644 index 00000000..30e25761 --- /dev/null +++ b/src/StellaOps.Feedser.WebService/StellaOps.Feedser.WebService.csproj @@ -0,0 +1,35 @@ + + + net10.0 + preview + enable + enable + true + StellaOps.Feedser.WebService + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/src/StellaOps.Feedser.WebService/TASKS.md b/src/StellaOps.Feedser.WebService/TASKS.md new file mode 100644 index 00000000..caeecae7 --- /dev/null +++ b/src/StellaOps.Feedser.WebService/TASKS.md @@ -0,0 +1,24 @@ +# TASKS +| Task | Owner(s) | Depends on | Notes | +|---|---|---|---| +|Bind & validate FeedserOptions|BE-Base|WebService|DONE – options bound/validated with failure logging.| +|Mongo service wiring|BE-Base|Storage.Mongo|DONE – wiring delegated to `AddMongoStorage`.| +|Bootstrapper execution on start|BE-Base|Storage.Mongo|DONE – startup calls `MongoBootstrapper.InitializeAsync`.| +|Plugin host options finalization|BE-Base|Plugins|DONE – default plugin directories/search patterns configured.| +|Jobs API contract tests|QA|Core|DONE – WebServiceEndpointsTests now cover success payloads, filtering, and trigger outcome mapping.| +|Health/Ready probes|DevOps|Ops|DONE – `/health` and `/ready` endpoints implemented.| +|Serilog + OTEL integration hooks|BE-Base|Observability|DONE – `TelemetryExtensions` wires Serilog + OTEL with configurable exporters.| +|Register built-in jobs (sources/exporters)|BE-Base|Core|DONE – AddBuiltInFeedserJobs adds fallback scheduler definitions for core connectors and exporters via reflection.| +|HTTP problem details consistency|BE-Base|WebService|DONE – API errors now emit RFC7807 responses with trace identifiers and typed problem categories.| +|Request logging and metrics|BE-Base|Observability|DONE – Serilog request logging enabled with enriched context and web.jobs counters published via OpenTelemetry.| +|Endpoint smoke tests (health/ready/jobs error paths)|QA|WebService|DONE – WebServiceEndpointsTests assert success and problem responses for health, ready, and job trigger error paths.| +|Batch job definition last-run lookup|BE-Base|Core|DONE – definitions endpoint now precomputes kinds array and reuses batched last-run dictionary; manual smoke verified via local GET `/jobs/definitions`.| +|Add no-cache headers to health/readiness/jobs APIs|BE-Base|WebService|DONE – helper applies Cache-Control/Pragma/Expires on all health/ready/jobs endpoints; awaiting automated probe tests once connector fixtures stabilize.| +|Authority configuration parity (FSR1)|DevEx/Feedser|Authority options schema|**DONE (2025-10-10)** – Options post-config loads clientSecretFile fallback, validators normalize scopes/audiences, and sample config documents issuer/credential/bypass settings.| +|Document authority toggle & scope requirements|Docs/Feedser|Authority integration|**DOING (2025-10-10)** – Quickstart updated with staging flag, client credentials, env overrides; operator guide refresh pending Docs guild review.| +|Plumb Authority client resilience options|BE-Base|Auth libraries LIB5|**TODO** – Bind retry/offline settings from the `authority` config block, flow them into `AddStellaOpsAuthClient`, and cover via WebService integration test.| +|Author ops guidance for resilience tuning|Docs/Feedser|Plumb Authority client resilience options|**TODO** – Extend operator/quickstart docs with recommended retry profiles, offline-tolerance guidance, and monitoring cues.| +|Document authority bypass logging patterns|Docs/Feedser|FSR3 logging|**TODO** – Capture new audit log fields (bypass, remote IP, subject) in operator docs and add troubleshooting guidance for cron bypasses.| +|Update Feedser operator guide for enforcement cutoff|Docs/Feedser|FSR1 rollout|**TODO** – Add `allowAnonymousFallback` sunset timeline and checklist to operator guide / runbooks before 2025-12-31 enforcement.| +|Rename plugin drop directory to namespaced path|BE-Base|Plugins|**TODO** – Point Feedser source/exporter build outputs to `StellaOps.Feedser.PluginBinaries`, update PluginHost defaults/search patterns to match, ensure Offline Kit packaging/tests expect the new folder, and document migration guidance for operators.| +|Authority resilience adoption|Feedser WebService, Docs|Plumb Authority client resilience options|**BLOCKED (2025-10-10)** – Roll out retry/offline knobs to deployment docs and confirm CLI parity once LIB5 lands; unblock after resilience options wired and tested.| diff --git a/src/StellaOps.Feedser.sln b/src/StellaOps.Feedser.sln new file mode 100644 index 00000000..775b223a --- /dev/null +++ b/src/StellaOps.Feedser.sln @@ -0,0 +1,916 @@ + +Microsoft Visual Studio Solution File, Format Version 12.00 +# Visual Studio Version 17 +VisualStudioVersion = 17.0.31903.59 +MinimumVisualStudioVersion = 10.0.40219.1 +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "StellaOps.Feedser.Source.Acsc", "StellaOps.Feedser.Source.Acsc\StellaOps.Feedser.Source.Acsc.csproj", "{CFD7B267-46B7-4C73-A33A-3E82AD2CFABC}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "StellaOps.Feedser.Source.Common", "StellaOps.Feedser.Source.Common\StellaOps.Feedser.Source.Common.csproj", "{E9DE840D-0760-4324-98E2-7F2CBE06DC1A}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "StellaOps.Feedser.Models", "StellaOps.Feedser.Models\StellaOps.Feedser.Models.csproj", "{061B0042-9A6C-4CFD-9E48-4D3F3B924442}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "StellaOps.Feedser.Source.Ics.Cisa", "StellaOps.Feedser.Source.Ics.Cisa\StellaOps.Feedser.Source.Ics.Cisa.csproj", "{6A301F32-2EEE-491B-9DB9-3BF26D032F07}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "StellaOps.Feedser.Core", "StellaOps.Feedser.Core\StellaOps.Feedser.Core.csproj", "{AFCCC916-58E8-4676-AABB-54B04CEA3392}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "StellaOps.Feedser.Storage.Mongo", "StellaOps.Feedser.Storage.Mongo\StellaOps.Feedser.Storage.Mongo.csproj", "{BF3DAB2F-E46E-49C1-9BA5-AA389763A632}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "StellaOps.Feedser.Normalization", "StellaOps.Feedser.Normalization\StellaOps.Feedser.Normalization.csproj", "{429BAA6A-706D-489A-846F-4B0EF1B15121}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "StellaOps.Feedser.Merge", "StellaOps.Feedser.Merge\StellaOps.Feedser.Merge.csproj", "{085CEC8E-0E10-48E8-89E2-9452CD2E7FA0}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "StellaOps.Feedser.Exporter.Json", "StellaOps.Feedser.Exporter.Json\StellaOps.Feedser.Exporter.Json.csproj", "{1C5506B8-C01B-4419-B888-A48F441E0C69}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "StellaOps.Feedser.Exporter.TrivyDb", "StellaOps.Feedser.Exporter.TrivyDb\StellaOps.Feedser.Exporter.TrivyDb.csproj", "{4D936BC4-5520-4642-A237-4106E97BC7A0}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "StellaOps.Plugin", "StellaOps.Plugin\StellaOps.Plugin.csproj", "{B85C1C0E-B245-44FB-877E-C112DE29041A}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "StellaOps.Feedser.WebService", "StellaOps.Feedser.WebService\StellaOps.Feedser.WebService.csproj", "{2C970A0F-FE3D-425B-B1B3-A008B194F5C2}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "StellaOps.Feedser.Source.Cccs", "StellaOps.Feedser.Source.Cccs\StellaOps.Feedser.Source.Cccs.csproj", "{A7035381-6D20-4A07-817B-A324ED735EB3}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "StellaOps.Feedser.Source.Distro.Debian", "StellaOps.Feedser.Source.Distro.Debian\StellaOps.Feedser.Source.Distro.Debian.csproj", "{404F5F6E-37E4-4EF9-B09D-6634366B5D44}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "StellaOps.Feedser.Source.Distro.Ubuntu", "StellaOps.Feedser.Source.Distro.Ubuntu\StellaOps.Feedser.Source.Distro.Ubuntu.csproj", "{1BEF4D9D-9EA4-4BE9-9664-F16DC1CA8EEB}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "StellaOps.Feedser.Source.Kisa", "StellaOps.Feedser.Source.Kisa\StellaOps.Feedser.Source.Kisa.csproj", "{23055A20-7079-4336-AD30-EFAA2FA11665}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "StellaOps.Feedser.Source.CertCc", "StellaOps.Feedser.Source.CertCc\StellaOps.Feedser.Source.CertCc.csproj", "{C2304954-9B15-4776-8DB6-22E293D311E4}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "StellaOps.Feedser.Source.CertFr", "StellaOps.Feedser.Source.CertFr\StellaOps.Feedser.Source.CertFr.csproj", "{E6895821-ED23-46D2-A5DC-06D61F90EC27}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "StellaOps.Feedser.Source.Nvd", "StellaOps.Feedser.Source.Nvd\StellaOps.Feedser.Source.Nvd.csproj", "{378CB675-D70B-4A95-B324-62B67D79AAB7}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "StellaOps.Feedser.Source.Vndr.Oracle", "StellaOps.Feedser.Source.Vndr.Oracle\StellaOps.Feedser.Source.Vndr.Oracle.csproj", "{53AD2E55-B0F5-46AD-BFE5-82F486371872}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "StellaOps.Feedser.Source.Ru.Nkcki", "StellaOps.Feedser.Source.Ru.Nkcki\StellaOps.Feedser.Source.Ru.Nkcki.csproj", "{B880C99C-C0BD-4953-95AD-2C76BC43F760}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "StellaOps.Feedser.Source.Distro.Suse", "StellaOps.Feedser.Source.Distro.Suse\StellaOps.Feedser.Source.Distro.Suse.csproj", "{23422F67-C1FB-4FF4-899C-706BCD63D9FD}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "StellaOps.Feedser.Source.Ru.Bdu", "StellaOps.Feedser.Source.Ru.Bdu\StellaOps.Feedser.Source.Ru.Bdu.csproj", "{16AD4AB9-2A80-4CFD-91A7-36CC1FEF439F}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "StellaOps.Feedser.Source.Kev", "StellaOps.Feedser.Source.Kev\StellaOps.Feedser.Source.Kev.csproj", "{20DB9837-715B-4515-98C6-14B50060B765}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "StellaOps.Feedser.Source.Ics.Kaspersky", "StellaOps.Feedser.Source.Ics.Kaspersky\StellaOps.Feedser.Source.Ics.Kaspersky.csproj", "{10849EE2-9F34-4C23-BBB4-916A59CDB7F4}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "StellaOps.Feedser.Source.Osv", "StellaOps.Feedser.Source.Osv\StellaOps.Feedser.Source.Osv.csproj", "{EFB16EDB-78D4-4601-852E-F4B37655FA13}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "StellaOps.Feedser.Source.Jvn", "StellaOps.Feedser.Source.Jvn\StellaOps.Feedser.Source.Jvn.csproj", "{02289F61-0173-42CC-B8F2-25CC53F8E066}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "StellaOps.Feedser.Source.CertBund", "StellaOps.Feedser.Source.CertBund\StellaOps.Feedser.Source.CertBund.csproj", "{4CE0B67B-2B6D-4D48-9D38-2F1165FD6BF4}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "StellaOps.Feedser.Source.Cve", "StellaOps.Feedser.Source.Cve\StellaOps.Feedser.Source.Cve.csproj", "{EB037D9A-EF9C-439D-8A79-4B7D12F9C9D0}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "StellaOps.Feedser.Source.Vndr.Cisco", "StellaOps.Feedser.Source.Vndr.Cisco\StellaOps.Feedser.Source.Vndr.Cisco.csproj", "{19957518-A422-4622-9FD1-621DF3E31869}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "StellaOps.Feedser.Source.Vndr.Msrc", "StellaOps.Feedser.Source.Vndr.Msrc\StellaOps.Feedser.Source.Vndr.Msrc.csproj", "{69C4C061-F5A0-4EAA-A4CD-9A513523952A}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "StellaOps.Feedser.Source.Vndr.Chromium", "StellaOps.Feedser.Source.Vndr.Chromium\StellaOps.Feedser.Source.Vndr.Chromium.csproj", "{C7F7DE6F-A369-4F43-9864-286DCEC615F8}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "StellaOps.Feedser.Source.Vndr.Apple", "StellaOps.Feedser.Source.Vndr.Apple\StellaOps.Feedser.Source.Vndr.Apple.csproj", "{1C1593FE-73A4-47E8-A45B-5FC3B0BA7698}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "StellaOps.Feedser.Source.Vndr.Vmware", "StellaOps.Feedser.Source.Vndr.Vmware\StellaOps.Feedser.Source.Vndr.Vmware.csproj", "{7255C38D-5A16-4A4D-98CE-CF0FD516B68E}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "StellaOps.Feedser.Source.Vndr.Adobe", "StellaOps.Feedser.Source.Vndr.Adobe\StellaOps.Feedser.Source.Vndr.Adobe.csproj", "{C3A42AA3-800D-4398-A077-5560EE6451EF}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "StellaOps.Feedser.Source.CertIn", "StellaOps.Feedser.Source.CertIn\StellaOps.Feedser.Source.CertIn.csproj", "{5016963A-6FC9-4063-AB83-2D1F9A2BC627}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "StellaOps.Feedser.Source.Ghsa", "StellaOps.Feedser.Source.Ghsa\StellaOps.Feedser.Source.Ghsa.csproj", "{72F43F43-F852-487F-8334-91D438CE2F7C}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "StellaOps.Feedser.Source.Distro.RedHat", "StellaOps.Feedser.Source.Distro.RedHat\StellaOps.Feedser.Source.Distro.RedHat.csproj", "{A4DBF88F-34D0-4A05-ACCE-DE080F912FDB}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "StellaOps.DependencyInjection", "StellaOps.DependencyInjection\StellaOps.DependencyInjection.csproj", "{F622D38D-DA49-473E-B724-E706F8113CF2}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "StellaOps.Feedser.Core.Tests", "StellaOps.Feedser.Core.Tests\StellaOps.Feedser.Core.Tests.csproj", "{3A3D7610-C864-4413-B07E-9E8C2A49A90E}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "StellaOps.Feedser.Merge.Tests", "StellaOps.Feedser.Merge.Tests\StellaOps.Feedser.Merge.Tests.csproj", "{9C4DEE96-CD7D-4AE3-A811-0B48B477003B}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "StellaOps.Feedser.Models.Tests", "StellaOps.Feedser.Models.Tests\StellaOps.Feedser.Models.Tests.csproj", "{437B2667-9461-47D2-B75B-4D2E03D69B94}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "StellaOps.Feedser.Normalization.Tests", "StellaOps.Feedser.Normalization.Tests\StellaOps.Feedser.Normalization.Tests.csproj", "{8249DF28-CDAF-4DEF-A912-C27F57B67FD5}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "StellaOps.Feedser.Storage.Mongo.Tests", "StellaOps.Feedser.Storage.Mongo.Tests\StellaOps.Feedser.Storage.Mongo.Tests.csproj", "{CBFB015B-C069-475F-A476-D52222729804}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "StellaOps.Feedser.Exporter.Json.Tests", "StellaOps.Feedser.Exporter.Json.Tests\StellaOps.Feedser.Exporter.Json.Tests.csproj", "{2A41D9D2-3218-4F12-9C2B-3DB18A8E732E}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "StellaOps.Feedser.Exporter.TrivyDb.Tests", "StellaOps.Feedser.Exporter.TrivyDb.Tests\StellaOps.Feedser.Exporter.TrivyDb.Tests.csproj", "{3EB22234-642E-4533-BCC3-93E8ED443B1D}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "StellaOps.Feedser.WebService.Tests", "StellaOps.Feedser.WebService.Tests\StellaOps.Feedser.WebService.Tests.csproj", "{84A5DE81-4444-499A-93BF-6DC4CA72F8D4}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "StellaOps.Feedser.Source.Common.Tests", "StellaOps.Feedser.Source.Common.Tests\StellaOps.Feedser.Source.Common.Tests.csproj", "{42E21E1D-C3DE-4765-93E9-39391BB5C802}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "StellaOps.Feedser.Source.Nvd.Tests", "StellaOps.Feedser.Source.Nvd.Tests\StellaOps.Feedser.Source.Nvd.Tests.csproj", "{B6E2EE26-B297-4AB9-A47E-A227F5EAE108}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "StellaOps.Feedser.Source.Distro.RedHat.Tests", "StellaOps.Feedser.Source.Distro.RedHat.Tests\StellaOps.Feedser.Source.Distro.RedHat.Tests.csproj", "{CDB2D636-C82F-43F1-BB30-FFC6258FBAB4}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "StellaOps.Feedser.Source.Vndr.Chromium.Tests", "StellaOps.Feedser.Source.Vndr.Chromium.Tests\StellaOps.Feedser.Source.Vndr.Chromium.Tests.csproj", "{2891FCDE-BB89-46F0-A40C-368EF804DB44}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "StellaOps.Feedser.Source.Vndr.Adobe.Tests", "StellaOps.Feedser.Source.Vndr.Adobe.Tests\StellaOps.Feedser.Source.Vndr.Adobe.Tests.csproj", "{B91C60FB-926F-47C3-BFD0-6DD145308344}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "StellaOps.Feedser.Source.Vndr.Oracle.Tests", "StellaOps.Feedser.Source.Vndr.Oracle.Tests\StellaOps.Feedser.Source.Vndr.Oracle.Tests.csproj", "{30DF89D1-D66D-4078-8A3B-951637A42265}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "StellaOps.Feedser.Source.Vndr.Vmware.Tests", "StellaOps.Feedser.Source.Vndr.Vmware.Tests\StellaOps.Feedser.Source.Vndr.Vmware.Tests.csproj", "{6E98C770-72FF-41FA-8C42-30AABAAF5B4E}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "StellaOps.Feedser.Source.CertIn.Tests", "StellaOps.Feedser.Source.CertIn.Tests\StellaOps.Feedser.Source.CertIn.Tests.csproj", "{79B36C92-BA93-4406-AB75-6F2282DDFF01}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "StellaOps.Feedser.Source.CertFr.Tests", "StellaOps.Feedser.Source.CertFr.Tests\StellaOps.Feedser.Source.CertFr.Tests.csproj", "{4B60FA53-81F6-4AB6-BE9F-DE0992E11977}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "StellaOps.Feedser.Source.Ics.Kaspersky.Tests", "StellaOps.Feedser.Source.Ics.Kaspersky.Tests\StellaOps.Feedser.Source.Ics.Kaspersky.Tests.csproj", "{6BBA820B-8443-4832-91C3-3AB002006494}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "StellaOps.Feedser.Source.Jvn.Tests", "StellaOps.Feedser.Source.Jvn.Tests\StellaOps.Feedser.Source.Jvn.Tests.csproj", "{7845AE1C-FBD7-4177-A06F-D7AAE8315DB2}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "StellaOps.Feedser.Source.Osv.Tests", "StellaOps.Feedser.Source.Osv.Tests\StellaOps.Feedser.Source.Osv.Tests.csproj", "{F892BFFD-9101-4D59-B6FD-C532EB04D51F}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "StellaOps.Feedser.Testing", "StellaOps.Feedser.Testing\StellaOps.Feedser.Testing.csproj", "{EAE910FC-188C-41C3-822A-623964CABE48}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "StellaOps.Feedser.Source.Distro.Debian.Tests", "StellaOps.Feedser.Source.Distro.Debian.Tests\StellaOps.Feedser.Source.Distro.Debian.Tests.csproj", "{BBA5C780-6348-427D-9600-726EAA8963B3}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "StellaOps.Configuration", "StellaOps.Configuration\StellaOps.Configuration.csproj", "{5F44A429-816A-4560-A5AA-61CD23FD8A19}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "StellaOps.Cli", "StellaOps.Cli\StellaOps.Cli.csproj", "{20FDC3B4-9908-4ABF-BA1D-50E0B4A64F4B}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "StellaOps.Cli.Tests", "StellaOps.Cli.Tests\StellaOps.Cli.Tests.csproj", "{544DBB82-4639-4856-A5F2-76828F7A8396}" +EndProject +Global + GlobalSection(SolutionConfigurationPlatforms) = preSolution + Debug|Any CPU = Debug|Any CPU + Debug|x64 = Debug|x64 + Debug|x86 = Debug|x86 + Release|Any CPU = Release|Any CPU + Release|x64 = Release|x64 + Release|x86 = Release|x86 + EndGlobalSection + GlobalSection(ProjectConfigurationPlatforms) = postSolution + {CFD7B267-46B7-4C73-A33A-3E82AD2CFABC}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {CFD7B267-46B7-4C73-A33A-3E82AD2CFABC}.Debug|Any CPU.Build.0 = Debug|Any CPU + {CFD7B267-46B7-4C73-A33A-3E82AD2CFABC}.Debug|x64.ActiveCfg = Debug|Any CPU + {CFD7B267-46B7-4C73-A33A-3E82AD2CFABC}.Debug|x64.Build.0 = Debug|Any CPU + {CFD7B267-46B7-4C73-A33A-3E82AD2CFABC}.Debug|x86.ActiveCfg = Debug|Any CPU + {CFD7B267-46B7-4C73-A33A-3E82AD2CFABC}.Debug|x86.Build.0 = Debug|Any CPU + {CFD7B267-46B7-4C73-A33A-3E82AD2CFABC}.Release|Any CPU.ActiveCfg = Release|Any CPU + {CFD7B267-46B7-4C73-A33A-3E82AD2CFABC}.Release|Any CPU.Build.0 = Release|Any CPU + {CFD7B267-46B7-4C73-A33A-3E82AD2CFABC}.Release|x64.ActiveCfg = Release|Any CPU + {CFD7B267-46B7-4C73-A33A-3E82AD2CFABC}.Release|x64.Build.0 = Release|Any CPU + {CFD7B267-46B7-4C73-A33A-3E82AD2CFABC}.Release|x86.ActiveCfg = Release|Any CPU + {CFD7B267-46B7-4C73-A33A-3E82AD2CFABC}.Release|x86.Build.0 = Release|Any CPU + {E9DE840D-0760-4324-98E2-7F2CBE06DC1A}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {E9DE840D-0760-4324-98E2-7F2CBE06DC1A}.Debug|Any CPU.Build.0 = Debug|Any CPU + {E9DE840D-0760-4324-98E2-7F2CBE06DC1A}.Debug|x64.ActiveCfg = Debug|Any CPU + {E9DE840D-0760-4324-98E2-7F2CBE06DC1A}.Debug|x64.Build.0 = Debug|Any CPU + {E9DE840D-0760-4324-98E2-7F2CBE06DC1A}.Debug|x86.ActiveCfg = Debug|Any CPU + {E9DE840D-0760-4324-98E2-7F2CBE06DC1A}.Debug|x86.Build.0 = Debug|Any CPU + {E9DE840D-0760-4324-98E2-7F2CBE06DC1A}.Release|Any CPU.ActiveCfg = Release|Any CPU + {E9DE840D-0760-4324-98E2-7F2CBE06DC1A}.Release|Any CPU.Build.0 = Release|Any CPU + {E9DE840D-0760-4324-98E2-7F2CBE06DC1A}.Release|x64.ActiveCfg = Release|Any CPU + {E9DE840D-0760-4324-98E2-7F2CBE06DC1A}.Release|x64.Build.0 = Release|Any CPU + {E9DE840D-0760-4324-98E2-7F2CBE06DC1A}.Release|x86.ActiveCfg = Release|Any CPU + {E9DE840D-0760-4324-98E2-7F2CBE06DC1A}.Release|x86.Build.0 = Release|Any CPU + {061B0042-9A6C-4CFD-9E48-4D3F3B924442}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {061B0042-9A6C-4CFD-9E48-4D3F3B924442}.Debug|Any CPU.Build.0 = Debug|Any CPU + {061B0042-9A6C-4CFD-9E48-4D3F3B924442}.Debug|x64.ActiveCfg = Debug|Any CPU + {061B0042-9A6C-4CFD-9E48-4D3F3B924442}.Debug|x64.Build.0 = Debug|Any CPU + {061B0042-9A6C-4CFD-9E48-4D3F3B924442}.Debug|x86.ActiveCfg = Debug|Any CPU + {061B0042-9A6C-4CFD-9E48-4D3F3B924442}.Debug|x86.Build.0 = Debug|Any CPU + {061B0042-9A6C-4CFD-9E48-4D3F3B924442}.Release|Any CPU.ActiveCfg = Release|Any CPU + {061B0042-9A6C-4CFD-9E48-4D3F3B924442}.Release|Any CPU.Build.0 = Release|Any CPU + {061B0042-9A6C-4CFD-9E48-4D3F3B924442}.Release|x64.ActiveCfg = Release|Any CPU + {061B0042-9A6C-4CFD-9E48-4D3F3B924442}.Release|x64.Build.0 = Release|Any CPU + {061B0042-9A6C-4CFD-9E48-4D3F3B924442}.Release|x86.ActiveCfg = Release|Any CPU + {061B0042-9A6C-4CFD-9E48-4D3F3B924442}.Release|x86.Build.0 = Release|Any CPU + {6A301F32-2EEE-491B-9DB9-3BF26D032F07}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {6A301F32-2EEE-491B-9DB9-3BF26D032F07}.Debug|Any CPU.Build.0 = Debug|Any CPU + {6A301F32-2EEE-491B-9DB9-3BF26D032F07}.Debug|x64.ActiveCfg = Debug|Any CPU + {6A301F32-2EEE-491B-9DB9-3BF26D032F07}.Debug|x64.Build.0 = Debug|Any CPU + {6A301F32-2EEE-491B-9DB9-3BF26D032F07}.Debug|x86.ActiveCfg = Debug|Any CPU + {6A301F32-2EEE-491B-9DB9-3BF26D032F07}.Debug|x86.Build.0 = Debug|Any CPU + {6A301F32-2EEE-491B-9DB9-3BF26D032F07}.Release|Any CPU.ActiveCfg = Release|Any CPU + {6A301F32-2EEE-491B-9DB9-3BF26D032F07}.Release|Any CPU.Build.0 = Release|Any CPU + {6A301F32-2EEE-491B-9DB9-3BF26D032F07}.Release|x64.ActiveCfg = Release|Any CPU + {6A301F32-2EEE-491B-9DB9-3BF26D032F07}.Release|x64.Build.0 = Release|Any CPU + {6A301F32-2EEE-491B-9DB9-3BF26D032F07}.Release|x86.ActiveCfg = Release|Any CPU + {6A301F32-2EEE-491B-9DB9-3BF26D032F07}.Release|x86.Build.0 = Release|Any CPU + {AFCCC916-58E8-4676-AABB-54B04CEA3392}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {AFCCC916-58E8-4676-AABB-54B04CEA3392}.Debug|Any CPU.Build.0 = Debug|Any CPU + {AFCCC916-58E8-4676-AABB-54B04CEA3392}.Debug|x64.ActiveCfg = Debug|Any CPU + {AFCCC916-58E8-4676-AABB-54B04CEA3392}.Debug|x64.Build.0 = Debug|Any CPU + {AFCCC916-58E8-4676-AABB-54B04CEA3392}.Debug|x86.ActiveCfg = Debug|Any CPU + {AFCCC916-58E8-4676-AABB-54B04CEA3392}.Debug|x86.Build.0 = Debug|Any CPU + {AFCCC916-58E8-4676-AABB-54B04CEA3392}.Release|Any CPU.ActiveCfg = Release|Any CPU + {AFCCC916-58E8-4676-AABB-54B04CEA3392}.Release|Any CPU.Build.0 = Release|Any CPU + {AFCCC916-58E8-4676-AABB-54B04CEA3392}.Release|x64.ActiveCfg = Release|Any CPU + {AFCCC916-58E8-4676-AABB-54B04CEA3392}.Release|x64.Build.0 = Release|Any CPU + {AFCCC916-58E8-4676-AABB-54B04CEA3392}.Release|x86.ActiveCfg = Release|Any CPU + {AFCCC916-58E8-4676-AABB-54B04CEA3392}.Release|x86.Build.0 = Release|Any CPU + {BF3DAB2F-E46E-49C1-9BA5-AA389763A632}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {BF3DAB2F-E46E-49C1-9BA5-AA389763A632}.Debug|Any CPU.Build.0 = Debug|Any CPU + {BF3DAB2F-E46E-49C1-9BA5-AA389763A632}.Debug|x64.ActiveCfg = Debug|Any CPU + {BF3DAB2F-E46E-49C1-9BA5-AA389763A632}.Debug|x64.Build.0 = Debug|Any CPU + {BF3DAB2F-E46E-49C1-9BA5-AA389763A632}.Debug|x86.ActiveCfg = Debug|Any CPU + {BF3DAB2F-E46E-49C1-9BA5-AA389763A632}.Debug|x86.Build.0 = Debug|Any CPU + {BF3DAB2F-E46E-49C1-9BA5-AA389763A632}.Release|Any CPU.ActiveCfg = Release|Any CPU + {BF3DAB2F-E46E-49C1-9BA5-AA389763A632}.Release|Any CPU.Build.0 = Release|Any CPU + {BF3DAB2F-E46E-49C1-9BA5-AA389763A632}.Release|x64.ActiveCfg = Release|Any CPU + {BF3DAB2F-E46E-49C1-9BA5-AA389763A632}.Release|x64.Build.0 = Release|Any CPU + {BF3DAB2F-E46E-49C1-9BA5-AA389763A632}.Release|x86.ActiveCfg = Release|Any CPU + {BF3DAB2F-E46E-49C1-9BA5-AA389763A632}.Release|x86.Build.0 = Release|Any CPU + {429BAA6A-706D-489A-846F-4B0EF1B15121}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {429BAA6A-706D-489A-846F-4B0EF1B15121}.Debug|Any CPU.Build.0 = Debug|Any CPU + {429BAA6A-706D-489A-846F-4B0EF1B15121}.Debug|x64.ActiveCfg = Debug|Any CPU + {429BAA6A-706D-489A-846F-4B0EF1B15121}.Debug|x64.Build.0 = Debug|Any CPU + {429BAA6A-706D-489A-846F-4B0EF1B15121}.Debug|x86.ActiveCfg = Debug|Any CPU + {429BAA6A-706D-489A-846F-4B0EF1B15121}.Debug|x86.Build.0 = Debug|Any CPU + {429BAA6A-706D-489A-846F-4B0EF1B15121}.Release|Any CPU.ActiveCfg = Release|Any CPU + {429BAA6A-706D-489A-846F-4B0EF1B15121}.Release|Any CPU.Build.0 = Release|Any CPU + {429BAA6A-706D-489A-846F-4B0EF1B15121}.Release|x64.ActiveCfg = Release|Any CPU + {429BAA6A-706D-489A-846F-4B0EF1B15121}.Release|x64.Build.0 = Release|Any CPU + {429BAA6A-706D-489A-846F-4B0EF1B15121}.Release|x86.ActiveCfg = Release|Any CPU + {429BAA6A-706D-489A-846F-4B0EF1B15121}.Release|x86.Build.0 = Release|Any CPU + {085CEC8E-0E10-48E8-89E2-9452CD2E7FA0}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {085CEC8E-0E10-48E8-89E2-9452CD2E7FA0}.Debug|Any CPU.Build.0 = Debug|Any CPU + {085CEC8E-0E10-48E8-89E2-9452CD2E7FA0}.Debug|x64.ActiveCfg = Debug|Any CPU + {085CEC8E-0E10-48E8-89E2-9452CD2E7FA0}.Debug|x64.Build.0 = Debug|Any CPU + {085CEC8E-0E10-48E8-89E2-9452CD2E7FA0}.Debug|x86.ActiveCfg = Debug|Any CPU + {085CEC8E-0E10-48E8-89E2-9452CD2E7FA0}.Debug|x86.Build.0 = Debug|Any CPU + {085CEC8E-0E10-48E8-89E2-9452CD2E7FA0}.Release|Any CPU.ActiveCfg = Release|Any CPU + {085CEC8E-0E10-48E8-89E2-9452CD2E7FA0}.Release|Any CPU.Build.0 = Release|Any CPU + {085CEC8E-0E10-48E8-89E2-9452CD2E7FA0}.Release|x64.ActiveCfg = Release|Any CPU + {085CEC8E-0E10-48E8-89E2-9452CD2E7FA0}.Release|x64.Build.0 = Release|Any CPU + {085CEC8E-0E10-48E8-89E2-9452CD2E7FA0}.Release|x86.ActiveCfg = Release|Any CPU + {085CEC8E-0E10-48E8-89E2-9452CD2E7FA0}.Release|x86.Build.0 = Release|Any CPU + {1C5506B8-C01B-4419-B888-A48F441E0C69}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {1C5506B8-C01B-4419-B888-A48F441E0C69}.Debug|Any CPU.Build.0 = Debug|Any CPU + {1C5506B8-C01B-4419-B888-A48F441E0C69}.Debug|x64.ActiveCfg = Debug|Any CPU + {1C5506B8-C01B-4419-B888-A48F441E0C69}.Debug|x64.Build.0 = Debug|Any CPU + {1C5506B8-C01B-4419-B888-A48F441E0C69}.Debug|x86.ActiveCfg = Debug|Any CPU + {1C5506B8-C01B-4419-B888-A48F441E0C69}.Debug|x86.Build.0 = Debug|Any CPU + {1C5506B8-C01B-4419-B888-A48F441E0C69}.Release|Any CPU.ActiveCfg = Release|Any CPU + {1C5506B8-C01B-4419-B888-A48F441E0C69}.Release|Any CPU.Build.0 = Release|Any CPU + {1C5506B8-C01B-4419-B888-A48F441E0C69}.Release|x64.ActiveCfg = Release|Any CPU + {1C5506B8-C01B-4419-B888-A48F441E0C69}.Release|x64.Build.0 = Release|Any CPU + {1C5506B8-C01B-4419-B888-A48F441E0C69}.Release|x86.ActiveCfg = Release|Any CPU + {1C5506B8-C01B-4419-B888-A48F441E0C69}.Release|x86.Build.0 = Release|Any CPU + {4D936BC4-5520-4642-A237-4106E97BC7A0}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {4D936BC4-5520-4642-A237-4106E97BC7A0}.Debug|Any CPU.Build.0 = Debug|Any CPU + {4D936BC4-5520-4642-A237-4106E97BC7A0}.Debug|x64.ActiveCfg = Debug|Any CPU + {4D936BC4-5520-4642-A237-4106E97BC7A0}.Debug|x64.Build.0 = Debug|Any CPU + {4D936BC4-5520-4642-A237-4106E97BC7A0}.Debug|x86.ActiveCfg = Debug|Any CPU + {4D936BC4-5520-4642-A237-4106E97BC7A0}.Debug|x86.Build.0 = Debug|Any CPU + {4D936BC4-5520-4642-A237-4106E97BC7A0}.Release|Any CPU.ActiveCfg = Release|Any CPU + {4D936BC4-5520-4642-A237-4106E97BC7A0}.Release|Any CPU.Build.0 = Release|Any CPU + {4D936BC4-5520-4642-A237-4106E97BC7A0}.Release|x64.ActiveCfg = Release|Any CPU + {4D936BC4-5520-4642-A237-4106E97BC7A0}.Release|x64.Build.0 = Release|Any CPU + {4D936BC4-5520-4642-A237-4106E97BC7A0}.Release|x86.ActiveCfg = Release|Any CPU + {4D936BC4-5520-4642-A237-4106E97BC7A0}.Release|x86.Build.0 = Release|Any CPU + {B85C1C0E-B245-44FB-877E-C112DE29041A}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {B85C1C0E-B245-44FB-877E-C112DE29041A}.Debug|Any CPU.Build.0 = Debug|Any CPU + {B85C1C0E-B245-44FB-877E-C112DE29041A}.Debug|x64.ActiveCfg = Debug|Any CPU + {B85C1C0E-B245-44FB-877E-C112DE29041A}.Debug|x64.Build.0 = Debug|Any CPU + {B85C1C0E-B245-44FB-877E-C112DE29041A}.Debug|x86.ActiveCfg = Debug|Any CPU + {B85C1C0E-B245-44FB-877E-C112DE29041A}.Debug|x86.Build.0 = Debug|Any CPU + {B85C1C0E-B245-44FB-877E-C112DE29041A}.Release|Any CPU.ActiveCfg = Release|Any CPU + {B85C1C0E-B245-44FB-877E-C112DE29041A}.Release|Any CPU.Build.0 = Release|Any CPU + {B85C1C0E-B245-44FB-877E-C112DE29041A}.Release|x64.ActiveCfg = Release|Any CPU + {B85C1C0E-B245-44FB-877E-C112DE29041A}.Release|x64.Build.0 = Release|Any CPU + {B85C1C0E-B245-44FB-877E-C112DE29041A}.Release|x86.ActiveCfg = Release|Any CPU + {B85C1C0E-B245-44FB-877E-C112DE29041A}.Release|x86.Build.0 = Release|Any CPU + {2C970A0F-FE3D-425B-B1B3-A008B194F5C2}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {2C970A0F-FE3D-425B-B1B3-A008B194F5C2}.Debug|Any CPU.Build.0 = Debug|Any CPU + {2C970A0F-FE3D-425B-B1B3-A008B194F5C2}.Debug|x64.ActiveCfg = Debug|Any CPU + {2C970A0F-FE3D-425B-B1B3-A008B194F5C2}.Debug|x64.Build.0 = Debug|Any CPU + {2C970A0F-FE3D-425B-B1B3-A008B194F5C2}.Debug|x86.ActiveCfg = Debug|Any CPU + {2C970A0F-FE3D-425B-B1B3-A008B194F5C2}.Debug|x86.Build.0 = Debug|Any CPU + {2C970A0F-FE3D-425B-B1B3-A008B194F5C2}.Release|Any CPU.ActiveCfg = Release|Any CPU + {2C970A0F-FE3D-425B-B1B3-A008B194F5C2}.Release|Any CPU.Build.0 = Release|Any CPU + {2C970A0F-FE3D-425B-B1B3-A008B194F5C2}.Release|x64.ActiveCfg = Release|Any CPU + {2C970A0F-FE3D-425B-B1B3-A008B194F5C2}.Release|x64.Build.0 = Release|Any CPU + {2C970A0F-FE3D-425B-B1B3-A008B194F5C2}.Release|x86.ActiveCfg = Release|Any CPU + {2C970A0F-FE3D-425B-B1B3-A008B194F5C2}.Release|x86.Build.0 = Release|Any CPU + {A7035381-6D20-4A07-817B-A324ED735EB3}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {A7035381-6D20-4A07-817B-A324ED735EB3}.Debug|Any CPU.Build.0 = Debug|Any CPU + {A7035381-6D20-4A07-817B-A324ED735EB3}.Debug|x64.ActiveCfg = Debug|Any CPU + {A7035381-6D20-4A07-817B-A324ED735EB3}.Debug|x64.Build.0 = Debug|Any CPU + {A7035381-6D20-4A07-817B-A324ED735EB3}.Debug|x86.ActiveCfg = Debug|Any CPU + {A7035381-6D20-4A07-817B-A324ED735EB3}.Debug|x86.Build.0 = Debug|Any CPU + {A7035381-6D20-4A07-817B-A324ED735EB3}.Release|Any CPU.ActiveCfg = Release|Any CPU + {A7035381-6D20-4A07-817B-A324ED735EB3}.Release|Any CPU.Build.0 = Release|Any CPU + {A7035381-6D20-4A07-817B-A324ED735EB3}.Release|x64.ActiveCfg = Release|Any CPU + {A7035381-6D20-4A07-817B-A324ED735EB3}.Release|x64.Build.0 = Release|Any CPU + {A7035381-6D20-4A07-817B-A324ED735EB3}.Release|x86.ActiveCfg = Release|Any CPU + {A7035381-6D20-4A07-817B-A324ED735EB3}.Release|x86.Build.0 = Release|Any CPU + {404F5F6E-37E4-4EF9-B09D-6634366B5D44}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {404F5F6E-37E4-4EF9-B09D-6634366B5D44}.Debug|Any CPU.Build.0 = Debug|Any CPU + {404F5F6E-37E4-4EF9-B09D-6634366B5D44}.Debug|x64.ActiveCfg = Debug|Any CPU + {404F5F6E-37E4-4EF9-B09D-6634366B5D44}.Debug|x64.Build.0 = Debug|Any CPU + {404F5F6E-37E4-4EF9-B09D-6634366B5D44}.Debug|x86.ActiveCfg = Debug|Any CPU + {404F5F6E-37E4-4EF9-B09D-6634366B5D44}.Debug|x86.Build.0 = Debug|Any CPU + {404F5F6E-37E4-4EF9-B09D-6634366B5D44}.Release|Any CPU.ActiveCfg = Release|Any CPU + {404F5F6E-37E4-4EF9-B09D-6634366B5D44}.Release|Any CPU.Build.0 = Release|Any CPU + {404F5F6E-37E4-4EF9-B09D-6634366B5D44}.Release|x64.ActiveCfg = Release|Any CPU + {404F5F6E-37E4-4EF9-B09D-6634366B5D44}.Release|x64.Build.0 = Release|Any CPU + {404F5F6E-37E4-4EF9-B09D-6634366B5D44}.Release|x86.ActiveCfg = Release|Any CPU + {404F5F6E-37E4-4EF9-B09D-6634366B5D44}.Release|x86.Build.0 = Release|Any CPU + {1BEF4D9D-9EA4-4BE9-9664-F16DC1CA8EEB}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {1BEF4D9D-9EA4-4BE9-9664-F16DC1CA8EEB}.Debug|Any CPU.Build.0 = Debug|Any CPU + {1BEF4D9D-9EA4-4BE9-9664-F16DC1CA8EEB}.Debug|x64.ActiveCfg = Debug|Any CPU + {1BEF4D9D-9EA4-4BE9-9664-F16DC1CA8EEB}.Debug|x64.Build.0 = Debug|Any CPU + {1BEF4D9D-9EA4-4BE9-9664-F16DC1CA8EEB}.Debug|x86.ActiveCfg = Debug|Any CPU + {1BEF4D9D-9EA4-4BE9-9664-F16DC1CA8EEB}.Debug|x86.Build.0 = Debug|Any CPU + {1BEF4D9D-9EA4-4BE9-9664-F16DC1CA8EEB}.Release|Any CPU.ActiveCfg = Release|Any CPU + {1BEF4D9D-9EA4-4BE9-9664-F16DC1CA8EEB}.Release|Any CPU.Build.0 = Release|Any CPU + {1BEF4D9D-9EA4-4BE9-9664-F16DC1CA8EEB}.Release|x64.ActiveCfg = Release|Any CPU + {1BEF4D9D-9EA4-4BE9-9664-F16DC1CA8EEB}.Release|x64.Build.0 = Release|Any CPU + {1BEF4D9D-9EA4-4BE9-9664-F16DC1CA8EEB}.Release|x86.ActiveCfg = Release|Any CPU + {1BEF4D9D-9EA4-4BE9-9664-F16DC1CA8EEB}.Release|x86.Build.0 = Release|Any CPU + {23055A20-7079-4336-AD30-EFAA2FA11665}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {23055A20-7079-4336-AD30-EFAA2FA11665}.Debug|Any CPU.Build.0 = Debug|Any CPU + {23055A20-7079-4336-AD30-EFAA2FA11665}.Debug|x64.ActiveCfg = Debug|Any CPU + {23055A20-7079-4336-AD30-EFAA2FA11665}.Debug|x64.Build.0 = Debug|Any CPU + {23055A20-7079-4336-AD30-EFAA2FA11665}.Debug|x86.ActiveCfg = Debug|Any CPU + {23055A20-7079-4336-AD30-EFAA2FA11665}.Debug|x86.Build.0 = Debug|Any CPU + {23055A20-7079-4336-AD30-EFAA2FA11665}.Release|Any CPU.ActiveCfg = Release|Any CPU + {23055A20-7079-4336-AD30-EFAA2FA11665}.Release|Any CPU.Build.0 = Release|Any CPU + {23055A20-7079-4336-AD30-EFAA2FA11665}.Release|x64.ActiveCfg = Release|Any CPU + {23055A20-7079-4336-AD30-EFAA2FA11665}.Release|x64.Build.0 = Release|Any CPU + {23055A20-7079-4336-AD30-EFAA2FA11665}.Release|x86.ActiveCfg = Release|Any CPU + {23055A20-7079-4336-AD30-EFAA2FA11665}.Release|x86.Build.0 = Release|Any CPU + {C2304954-9B15-4776-8DB6-22E293D311E4}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {C2304954-9B15-4776-8DB6-22E293D311E4}.Debug|Any CPU.Build.0 = Debug|Any CPU + {C2304954-9B15-4776-8DB6-22E293D311E4}.Debug|x64.ActiveCfg = Debug|Any CPU + {C2304954-9B15-4776-8DB6-22E293D311E4}.Debug|x64.Build.0 = Debug|Any CPU + {C2304954-9B15-4776-8DB6-22E293D311E4}.Debug|x86.ActiveCfg = Debug|Any CPU + {C2304954-9B15-4776-8DB6-22E293D311E4}.Debug|x86.Build.0 = Debug|Any CPU + {C2304954-9B15-4776-8DB6-22E293D311E4}.Release|Any CPU.ActiveCfg = Release|Any CPU + {C2304954-9B15-4776-8DB6-22E293D311E4}.Release|Any CPU.Build.0 = Release|Any CPU + {C2304954-9B15-4776-8DB6-22E293D311E4}.Release|x64.ActiveCfg = Release|Any CPU + {C2304954-9B15-4776-8DB6-22E293D311E4}.Release|x64.Build.0 = Release|Any CPU + {C2304954-9B15-4776-8DB6-22E293D311E4}.Release|x86.ActiveCfg = Release|Any CPU + {C2304954-9B15-4776-8DB6-22E293D311E4}.Release|x86.Build.0 = Release|Any CPU + {E6895821-ED23-46D2-A5DC-06D61F90EC27}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {E6895821-ED23-46D2-A5DC-06D61F90EC27}.Debug|Any CPU.Build.0 = Debug|Any CPU + {E6895821-ED23-46D2-A5DC-06D61F90EC27}.Debug|x64.ActiveCfg = Debug|Any CPU + {E6895821-ED23-46D2-A5DC-06D61F90EC27}.Debug|x64.Build.0 = Debug|Any CPU + {E6895821-ED23-46D2-A5DC-06D61F90EC27}.Debug|x86.ActiveCfg = Debug|Any CPU + {E6895821-ED23-46D2-A5DC-06D61F90EC27}.Debug|x86.Build.0 = Debug|Any CPU + {E6895821-ED23-46D2-A5DC-06D61F90EC27}.Release|Any CPU.ActiveCfg = Release|Any CPU + {E6895821-ED23-46D2-A5DC-06D61F90EC27}.Release|Any CPU.Build.0 = Release|Any CPU + {E6895821-ED23-46D2-A5DC-06D61F90EC27}.Release|x64.ActiveCfg = Release|Any CPU + {E6895821-ED23-46D2-A5DC-06D61F90EC27}.Release|x64.Build.0 = Release|Any CPU + {E6895821-ED23-46D2-A5DC-06D61F90EC27}.Release|x86.ActiveCfg = Release|Any CPU + {E6895821-ED23-46D2-A5DC-06D61F90EC27}.Release|x86.Build.0 = Release|Any CPU + {378CB675-D70B-4A95-B324-62B67D79AAB7}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {378CB675-D70B-4A95-B324-62B67D79AAB7}.Debug|Any CPU.Build.0 = Debug|Any CPU + {378CB675-D70B-4A95-B324-62B67D79AAB7}.Debug|x64.ActiveCfg = Debug|Any CPU + {378CB675-D70B-4A95-B324-62B67D79AAB7}.Debug|x64.Build.0 = Debug|Any CPU + {378CB675-D70B-4A95-B324-62B67D79AAB7}.Debug|x86.ActiveCfg = Debug|Any CPU + {378CB675-D70B-4A95-B324-62B67D79AAB7}.Debug|x86.Build.0 = Debug|Any CPU + {378CB675-D70B-4A95-B324-62B67D79AAB7}.Release|Any CPU.ActiveCfg = Release|Any CPU + {378CB675-D70B-4A95-B324-62B67D79AAB7}.Release|Any CPU.Build.0 = Release|Any CPU + {378CB675-D70B-4A95-B324-62B67D79AAB7}.Release|x64.ActiveCfg = Release|Any CPU + {378CB675-D70B-4A95-B324-62B67D79AAB7}.Release|x64.Build.0 = Release|Any CPU + {378CB675-D70B-4A95-B324-62B67D79AAB7}.Release|x86.ActiveCfg = Release|Any CPU + {378CB675-D70B-4A95-B324-62B67D79AAB7}.Release|x86.Build.0 = Release|Any CPU + {53AD2E55-B0F5-46AD-BFE5-82F486371872}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {53AD2E55-B0F5-46AD-BFE5-82F486371872}.Debug|Any CPU.Build.0 = Debug|Any CPU + {53AD2E55-B0F5-46AD-BFE5-82F486371872}.Debug|x64.ActiveCfg = Debug|Any CPU + {53AD2E55-B0F5-46AD-BFE5-82F486371872}.Debug|x64.Build.0 = Debug|Any CPU + {53AD2E55-B0F5-46AD-BFE5-82F486371872}.Debug|x86.ActiveCfg = Debug|Any CPU + {53AD2E55-B0F5-46AD-BFE5-82F486371872}.Debug|x86.Build.0 = Debug|Any CPU + {53AD2E55-B0F5-46AD-BFE5-82F486371872}.Release|Any CPU.ActiveCfg = Release|Any CPU + {53AD2E55-B0F5-46AD-BFE5-82F486371872}.Release|Any CPU.Build.0 = Release|Any CPU + {53AD2E55-B0F5-46AD-BFE5-82F486371872}.Release|x64.ActiveCfg = Release|Any CPU + {53AD2E55-B0F5-46AD-BFE5-82F486371872}.Release|x64.Build.0 = Release|Any CPU + {53AD2E55-B0F5-46AD-BFE5-82F486371872}.Release|x86.ActiveCfg = Release|Any CPU + {53AD2E55-B0F5-46AD-BFE5-82F486371872}.Release|x86.Build.0 = Release|Any CPU + {B880C99C-C0BD-4953-95AD-2C76BC43F760}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {B880C99C-C0BD-4953-95AD-2C76BC43F760}.Debug|Any CPU.Build.0 = Debug|Any CPU + {B880C99C-C0BD-4953-95AD-2C76BC43F760}.Debug|x64.ActiveCfg = Debug|Any CPU + {B880C99C-C0BD-4953-95AD-2C76BC43F760}.Debug|x64.Build.0 = Debug|Any CPU + {B880C99C-C0BD-4953-95AD-2C76BC43F760}.Debug|x86.ActiveCfg = Debug|Any CPU + {B880C99C-C0BD-4953-95AD-2C76BC43F760}.Debug|x86.Build.0 = Debug|Any CPU + {B880C99C-C0BD-4953-95AD-2C76BC43F760}.Release|Any CPU.ActiveCfg = Release|Any CPU + {B880C99C-C0BD-4953-95AD-2C76BC43F760}.Release|Any CPU.Build.0 = Release|Any CPU + {B880C99C-C0BD-4953-95AD-2C76BC43F760}.Release|x64.ActiveCfg = Release|Any CPU + {B880C99C-C0BD-4953-95AD-2C76BC43F760}.Release|x64.Build.0 = Release|Any CPU + {B880C99C-C0BD-4953-95AD-2C76BC43F760}.Release|x86.ActiveCfg = Release|Any CPU + {B880C99C-C0BD-4953-95AD-2C76BC43F760}.Release|x86.Build.0 = Release|Any CPU + {23422F67-C1FB-4FF4-899C-706BCD63D9FD}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {23422F67-C1FB-4FF4-899C-706BCD63D9FD}.Debug|Any CPU.Build.0 = Debug|Any CPU + {23422F67-C1FB-4FF4-899C-706BCD63D9FD}.Debug|x64.ActiveCfg = Debug|Any CPU + {23422F67-C1FB-4FF4-899C-706BCD63D9FD}.Debug|x64.Build.0 = Debug|Any CPU + {23422F67-C1FB-4FF4-899C-706BCD63D9FD}.Debug|x86.ActiveCfg = Debug|Any CPU + {23422F67-C1FB-4FF4-899C-706BCD63D9FD}.Debug|x86.Build.0 = Debug|Any CPU + {23422F67-C1FB-4FF4-899C-706BCD63D9FD}.Release|Any CPU.ActiveCfg = Release|Any CPU + {23422F67-C1FB-4FF4-899C-706BCD63D9FD}.Release|Any CPU.Build.0 = Release|Any CPU + {23422F67-C1FB-4FF4-899C-706BCD63D9FD}.Release|x64.ActiveCfg = Release|Any CPU + {23422F67-C1FB-4FF4-899C-706BCD63D9FD}.Release|x64.Build.0 = Release|Any CPU + {23422F67-C1FB-4FF4-899C-706BCD63D9FD}.Release|x86.ActiveCfg = Release|Any CPU + {23422F67-C1FB-4FF4-899C-706BCD63D9FD}.Release|x86.Build.0 = Release|Any CPU + {16AD4AB9-2A80-4CFD-91A7-36CC1FEF439F}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {16AD4AB9-2A80-4CFD-91A7-36CC1FEF439F}.Debug|Any CPU.Build.0 = Debug|Any CPU + {16AD4AB9-2A80-4CFD-91A7-36CC1FEF439F}.Debug|x64.ActiveCfg = Debug|Any CPU + {16AD4AB9-2A80-4CFD-91A7-36CC1FEF439F}.Debug|x64.Build.0 = Debug|Any CPU + {16AD4AB9-2A80-4CFD-91A7-36CC1FEF439F}.Debug|x86.ActiveCfg = Debug|Any CPU + {16AD4AB9-2A80-4CFD-91A7-36CC1FEF439F}.Debug|x86.Build.0 = Debug|Any CPU + {16AD4AB9-2A80-4CFD-91A7-36CC1FEF439F}.Release|Any CPU.ActiveCfg = Release|Any CPU + {16AD4AB9-2A80-4CFD-91A7-36CC1FEF439F}.Release|Any CPU.Build.0 = Release|Any CPU + {16AD4AB9-2A80-4CFD-91A7-36CC1FEF439F}.Release|x64.ActiveCfg = Release|Any CPU + {16AD4AB9-2A80-4CFD-91A7-36CC1FEF439F}.Release|x64.Build.0 = Release|Any CPU + {16AD4AB9-2A80-4CFD-91A7-36CC1FEF439F}.Release|x86.ActiveCfg = Release|Any CPU + {16AD4AB9-2A80-4CFD-91A7-36CC1FEF439F}.Release|x86.Build.0 = Release|Any CPU + {20DB9837-715B-4515-98C6-14B50060B765}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {20DB9837-715B-4515-98C6-14B50060B765}.Debug|Any CPU.Build.0 = Debug|Any CPU + {20DB9837-715B-4515-98C6-14B50060B765}.Debug|x64.ActiveCfg = Debug|Any CPU + {20DB9837-715B-4515-98C6-14B50060B765}.Debug|x64.Build.0 = Debug|Any CPU + {20DB9837-715B-4515-98C6-14B50060B765}.Debug|x86.ActiveCfg = Debug|Any CPU + {20DB9837-715B-4515-98C6-14B50060B765}.Debug|x86.Build.0 = Debug|Any CPU + {20DB9837-715B-4515-98C6-14B50060B765}.Release|Any CPU.ActiveCfg = Release|Any CPU + {20DB9837-715B-4515-98C6-14B50060B765}.Release|Any CPU.Build.0 = Release|Any CPU + {20DB9837-715B-4515-98C6-14B50060B765}.Release|x64.ActiveCfg = Release|Any CPU + {20DB9837-715B-4515-98C6-14B50060B765}.Release|x64.Build.0 = Release|Any CPU + {20DB9837-715B-4515-98C6-14B50060B765}.Release|x86.ActiveCfg = Release|Any CPU + {20DB9837-715B-4515-98C6-14B50060B765}.Release|x86.Build.0 = Release|Any CPU + {10849EE2-9F34-4C23-BBB4-916A59CDB7F4}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {10849EE2-9F34-4C23-BBB4-916A59CDB7F4}.Debug|Any CPU.Build.0 = Debug|Any CPU + {10849EE2-9F34-4C23-BBB4-916A59CDB7F4}.Debug|x64.ActiveCfg = Debug|Any CPU + {10849EE2-9F34-4C23-BBB4-916A59CDB7F4}.Debug|x64.Build.0 = Debug|Any CPU + {10849EE2-9F34-4C23-BBB4-916A59CDB7F4}.Debug|x86.ActiveCfg = Debug|Any CPU + {10849EE2-9F34-4C23-BBB4-916A59CDB7F4}.Debug|x86.Build.0 = Debug|Any CPU + {10849EE2-9F34-4C23-BBB4-916A59CDB7F4}.Release|Any CPU.ActiveCfg = Release|Any CPU + {10849EE2-9F34-4C23-BBB4-916A59CDB7F4}.Release|Any CPU.Build.0 = Release|Any CPU + {10849EE2-9F34-4C23-BBB4-916A59CDB7F4}.Release|x64.ActiveCfg = Release|Any CPU + {10849EE2-9F34-4C23-BBB4-916A59CDB7F4}.Release|x64.Build.0 = Release|Any CPU + {10849EE2-9F34-4C23-BBB4-916A59CDB7F4}.Release|x86.ActiveCfg = Release|Any CPU + {10849EE2-9F34-4C23-BBB4-916A59CDB7F4}.Release|x86.Build.0 = Release|Any CPU + {EFB16EDB-78D4-4601-852E-F4B37655FA13}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {EFB16EDB-78D4-4601-852E-F4B37655FA13}.Debug|Any CPU.Build.0 = Debug|Any CPU + {EFB16EDB-78D4-4601-852E-F4B37655FA13}.Debug|x64.ActiveCfg = Debug|Any CPU + {EFB16EDB-78D4-4601-852E-F4B37655FA13}.Debug|x64.Build.0 = Debug|Any CPU + {EFB16EDB-78D4-4601-852E-F4B37655FA13}.Debug|x86.ActiveCfg = Debug|Any CPU + {EFB16EDB-78D4-4601-852E-F4B37655FA13}.Debug|x86.Build.0 = Debug|Any CPU + {EFB16EDB-78D4-4601-852E-F4B37655FA13}.Release|Any CPU.ActiveCfg = Release|Any CPU + {EFB16EDB-78D4-4601-852E-F4B37655FA13}.Release|Any CPU.Build.0 = Release|Any CPU + {EFB16EDB-78D4-4601-852E-F4B37655FA13}.Release|x64.ActiveCfg = Release|Any CPU + {EFB16EDB-78D4-4601-852E-F4B37655FA13}.Release|x64.Build.0 = Release|Any CPU + {EFB16EDB-78D4-4601-852E-F4B37655FA13}.Release|x86.ActiveCfg = Release|Any CPU + {EFB16EDB-78D4-4601-852E-F4B37655FA13}.Release|x86.Build.0 = Release|Any CPU + {02289F61-0173-42CC-B8F2-25CC53F8E066}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {02289F61-0173-42CC-B8F2-25CC53F8E066}.Debug|Any CPU.Build.0 = Debug|Any CPU + {02289F61-0173-42CC-B8F2-25CC53F8E066}.Debug|x64.ActiveCfg = Debug|Any CPU + {02289F61-0173-42CC-B8F2-25CC53F8E066}.Debug|x64.Build.0 = Debug|Any CPU + {02289F61-0173-42CC-B8F2-25CC53F8E066}.Debug|x86.ActiveCfg = Debug|Any CPU + {02289F61-0173-42CC-B8F2-25CC53F8E066}.Debug|x86.Build.0 = Debug|Any CPU + {02289F61-0173-42CC-B8F2-25CC53F8E066}.Release|Any CPU.ActiveCfg = Release|Any CPU + {02289F61-0173-42CC-B8F2-25CC53F8E066}.Release|Any CPU.Build.0 = Release|Any CPU + {02289F61-0173-42CC-B8F2-25CC53F8E066}.Release|x64.ActiveCfg = Release|Any CPU + {02289F61-0173-42CC-B8F2-25CC53F8E066}.Release|x64.Build.0 = Release|Any CPU + {02289F61-0173-42CC-B8F2-25CC53F8E066}.Release|x86.ActiveCfg = Release|Any CPU + {02289F61-0173-42CC-B8F2-25CC53F8E066}.Release|x86.Build.0 = Release|Any CPU + {4CE0B67B-2B6D-4D48-9D38-2F1165FD6BF4}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {4CE0B67B-2B6D-4D48-9D38-2F1165FD6BF4}.Debug|Any CPU.Build.0 = Debug|Any CPU + {4CE0B67B-2B6D-4D48-9D38-2F1165FD6BF4}.Debug|x64.ActiveCfg = Debug|Any CPU + {4CE0B67B-2B6D-4D48-9D38-2F1165FD6BF4}.Debug|x64.Build.0 = Debug|Any CPU + {4CE0B67B-2B6D-4D48-9D38-2F1165FD6BF4}.Debug|x86.ActiveCfg = Debug|Any CPU + {4CE0B67B-2B6D-4D48-9D38-2F1165FD6BF4}.Debug|x86.Build.0 = Debug|Any CPU + {4CE0B67B-2B6D-4D48-9D38-2F1165FD6BF4}.Release|Any CPU.ActiveCfg = Release|Any CPU + {4CE0B67B-2B6D-4D48-9D38-2F1165FD6BF4}.Release|Any CPU.Build.0 = Release|Any CPU + {4CE0B67B-2B6D-4D48-9D38-2F1165FD6BF4}.Release|x64.ActiveCfg = Release|Any CPU + {4CE0B67B-2B6D-4D48-9D38-2F1165FD6BF4}.Release|x64.Build.0 = Release|Any CPU + {4CE0B67B-2B6D-4D48-9D38-2F1165FD6BF4}.Release|x86.ActiveCfg = Release|Any CPU + {4CE0B67B-2B6D-4D48-9D38-2F1165FD6BF4}.Release|x86.Build.0 = Release|Any CPU + {EB037D9A-EF9C-439D-8A79-4B7D12F9C9D0}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {EB037D9A-EF9C-439D-8A79-4B7D12F9C9D0}.Debug|Any CPU.Build.0 = Debug|Any CPU + {EB037D9A-EF9C-439D-8A79-4B7D12F9C9D0}.Debug|x64.ActiveCfg = Debug|Any CPU + {EB037D9A-EF9C-439D-8A79-4B7D12F9C9D0}.Debug|x64.Build.0 = Debug|Any CPU + {EB037D9A-EF9C-439D-8A79-4B7D12F9C9D0}.Debug|x86.ActiveCfg = Debug|Any CPU + {EB037D9A-EF9C-439D-8A79-4B7D12F9C9D0}.Debug|x86.Build.0 = Debug|Any CPU + {EB037D9A-EF9C-439D-8A79-4B7D12F9C9D0}.Release|Any CPU.ActiveCfg = Release|Any CPU + {EB037D9A-EF9C-439D-8A79-4B7D12F9C9D0}.Release|Any CPU.Build.0 = Release|Any CPU + {EB037D9A-EF9C-439D-8A79-4B7D12F9C9D0}.Release|x64.ActiveCfg = Release|Any CPU + {EB037D9A-EF9C-439D-8A79-4B7D12F9C9D0}.Release|x64.Build.0 = Release|Any CPU + {EB037D9A-EF9C-439D-8A79-4B7D12F9C9D0}.Release|x86.ActiveCfg = Release|Any CPU + {EB037D9A-EF9C-439D-8A79-4B7D12F9C9D0}.Release|x86.Build.0 = Release|Any CPU + {19957518-A422-4622-9FD1-621DF3E31869}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {19957518-A422-4622-9FD1-621DF3E31869}.Debug|Any CPU.Build.0 = Debug|Any CPU + {19957518-A422-4622-9FD1-621DF3E31869}.Debug|x64.ActiveCfg = Debug|Any CPU + {19957518-A422-4622-9FD1-621DF3E31869}.Debug|x64.Build.0 = Debug|Any CPU + {19957518-A422-4622-9FD1-621DF3E31869}.Debug|x86.ActiveCfg = Debug|Any CPU + {19957518-A422-4622-9FD1-621DF3E31869}.Debug|x86.Build.0 = Debug|Any CPU + {19957518-A422-4622-9FD1-621DF3E31869}.Release|Any CPU.ActiveCfg = Release|Any CPU + {19957518-A422-4622-9FD1-621DF3E31869}.Release|Any CPU.Build.0 = Release|Any CPU + {19957518-A422-4622-9FD1-621DF3E31869}.Release|x64.ActiveCfg = Release|Any CPU + {19957518-A422-4622-9FD1-621DF3E31869}.Release|x64.Build.0 = Release|Any CPU + {19957518-A422-4622-9FD1-621DF3E31869}.Release|x86.ActiveCfg = Release|Any CPU + {19957518-A422-4622-9FD1-621DF3E31869}.Release|x86.Build.0 = Release|Any CPU + {69C4C061-F5A0-4EAA-A4CD-9A513523952A}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {69C4C061-F5A0-4EAA-A4CD-9A513523952A}.Debug|Any CPU.Build.0 = Debug|Any CPU + {69C4C061-F5A0-4EAA-A4CD-9A513523952A}.Debug|x64.ActiveCfg = Debug|Any CPU + {69C4C061-F5A0-4EAA-A4CD-9A513523952A}.Debug|x64.Build.0 = Debug|Any CPU + {69C4C061-F5A0-4EAA-A4CD-9A513523952A}.Debug|x86.ActiveCfg = Debug|Any CPU + {69C4C061-F5A0-4EAA-A4CD-9A513523952A}.Debug|x86.Build.0 = Debug|Any CPU + {69C4C061-F5A0-4EAA-A4CD-9A513523952A}.Release|Any CPU.ActiveCfg = Release|Any CPU + {69C4C061-F5A0-4EAA-A4CD-9A513523952A}.Release|Any CPU.Build.0 = Release|Any CPU + {69C4C061-F5A0-4EAA-A4CD-9A513523952A}.Release|x64.ActiveCfg = Release|Any CPU + {69C4C061-F5A0-4EAA-A4CD-9A513523952A}.Release|x64.Build.0 = Release|Any CPU + {69C4C061-F5A0-4EAA-A4CD-9A513523952A}.Release|x86.ActiveCfg = Release|Any CPU + {69C4C061-F5A0-4EAA-A4CD-9A513523952A}.Release|x86.Build.0 = Release|Any CPU + {C7F7DE6F-A369-4F43-9864-286DCEC615F8}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {C7F7DE6F-A369-4F43-9864-286DCEC615F8}.Debug|Any CPU.Build.0 = Debug|Any CPU + {C7F7DE6F-A369-4F43-9864-286DCEC615F8}.Debug|x64.ActiveCfg = Debug|Any CPU + {C7F7DE6F-A369-4F43-9864-286DCEC615F8}.Debug|x64.Build.0 = Debug|Any CPU + {C7F7DE6F-A369-4F43-9864-286DCEC615F8}.Debug|x86.ActiveCfg = Debug|Any CPU + {C7F7DE6F-A369-4F43-9864-286DCEC615F8}.Debug|x86.Build.0 = Debug|Any CPU + {C7F7DE6F-A369-4F43-9864-286DCEC615F8}.Release|Any CPU.ActiveCfg = Release|Any CPU + {C7F7DE6F-A369-4F43-9864-286DCEC615F8}.Release|Any CPU.Build.0 = Release|Any CPU + {C7F7DE6F-A369-4F43-9864-286DCEC615F8}.Release|x64.ActiveCfg = Release|Any CPU + {C7F7DE6F-A369-4F43-9864-286DCEC615F8}.Release|x64.Build.0 = Release|Any CPU + {C7F7DE6F-A369-4F43-9864-286DCEC615F8}.Release|x86.ActiveCfg = Release|Any CPU + {C7F7DE6F-A369-4F43-9864-286DCEC615F8}.Release|x86.Build.0 = Release|Any CPU + {1C1593FE-73A4-47E8-A45B-5FC3B0BA7698}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {1C1593FE-73A4-47E8-A45B-5FC3B0BA7698}.Debug|Any CPU.Build.0 = Debug|Any CPU + {1C1593FE-73A4-47E8-A45B-5FC3B0BA7698}.Debug|x64.ActiveCfg = Debug|Any CPU + {1C1593FE-73A4-47E8-A45B-5FC3B0BA7698}.Debug|x64.Build.0 = Debug|Any CPU + {1C1593FE-73A4-47E8-A45B-5FC3B0BA7698}.Debug|x86.ActiveCfg = Debug|Any CPU + {1C1593FE-73A4-47E8-A45B-5FC3B0BA7698}.Debug|x86.Build.0 = Debug|Any CPU + {1C1593FE-73A4-47E8-A45B-5FC3B0BA7698}.Release|Any CPU.ActiveCfg = Release|Any CPU + {1C1593FE-73A4-47E8-A45B-5FC3B0BA7698}.Release|Any CPU.Build.0 = Release|Any CPU + {1C1593FE-73A4-47E8-A45B-5FC3B0BA7698}.Release|x64.ActiveCfg = Release|Any CPU + {1C1593FE-73A4-47E8-A45B-5FC3B0BA7698}.Release|x64.Build.0 = Release|Any CPU + {1C1593FE-73A4-47E8-A45B-5FC3B0BA7698}.Release|x86.ActiveCfg = Release|Any CPU + {1C1593FE-73A4-47E8-A45B-5FC3B0BA7698}.Release|x86.Build.0 = Release|Any CPU + {7255C38D-5A16-4A4D-98CE-CF0FD516B68E}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {7255C38D-5A16-4A4D-98CE-CF0FD516B68E}.Debug|Any CPU.Build.0 = Debug|Any CPU + {7255C38D-5A16-4A4D-98CE-CF0FD516B68E}.Debug|x64.ActiveCfg = Debug|Any CPU + {7255C38D-5A16-4A4D-98CE-CF0FD516B68E}.Debug|x64.Build.0 = Debug|Any CPU + {7255C38D-5A16-4A4D-98CE-CF0FD516B68E}.Debug|x86.ActiveCfg = Debug|Any CPU + {7255C38D-5A16-4A4D-98CE-CF0FD516B68E}.Debug|x86.Build.0 = Debug|Any CPU + {7255C38D-5A16-4A4D-98CE-CF0FD516B68E}.Release|Any CPU.ActiveCfg = Release|Any CPU + {7255C38D-5A16-4A4D-98CE-CF0FD516B68E}.Release|Any CPU.Build.0 = Release|Any CPU + {7255C38D-5A16-4A4D-98CE-CF0FD516B68E}.Release|x64.ActiveCfg = Release|Any CPU + {7255C38D-5A16-4A4D-98CE-CF0FD516B68E}.Release|x64.Build.0 = Release|Any CPU + {7255C38D-5A16-4A4D-98CE-CF0FD516B68E}.Release|x86.ActiveCfg = Release|Any CPU + {7255C38D-5A16-4A4D-98CE-CF0FD516B68E}.Release|x86.Build.0 = Release|Any CPU + {C3A42AA3-800D-4398-A077-5560EE6451EF}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {C3A42AA3-800D-4398-A077-5560EE6451EF}.Debug|Any CPU.Build.0 = Debug|Any CPU + {C3A42AA3-800D-4398-A077-5560EE6451EF}.Debug|x64.ActiveCfg = Debug|Any CPU + {C3A42AA3-800D-4398-A077-5560EE6451EF}.Debug|x64.Build.0 = Debug|Any CPU + {C3A42AA3-800D-4398-A077-5560EE6451EF}.Debug|x86.ActiveCfg = Debug|Any CPU + {C3A42AA3-800D-4398-A077-5560EE6451EF}.Debug|x86.Build.0 = Debug|Any CPU + {C3A42AA3-800D-4398-A077-5560EE6451EF}.Release|Any CPU.ActiveCfg = Release|Any CPU + {C3A42AA3-800D-4398-A077-5560EE6451EF}.Release|Any CPU.Build.0 = Release|Any CPU + {C3A42AA3-800D-4398-A077-5560EE6451EF}.Release|x64.ActiveCfg = Release|Any CPU + {C3A42AA3-800D-4398-A077-5560EE6451EF}.Release|x64.Build.0 = Release|Any CPU + {C3A42AA3-800D-4398-A077-5560EE6451EF}.Release|x86.ActiveCfg = Release|Any CPU + {C3A42AA3-800D-4398-A077-5560EE6451EF}.Release|x86.Build.0 = Release|Any CPU + {5016963A-6FC9-4063-AB83-2D1F9A2BC627}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {5016963A-6FC9-4063-AB83-2D1F9A2BC627}.Debug|Any CPU.Build.0 = Debug|Any CPU + {5016963A-6FC9-4063-AB83-2D1F9A2BC627}.Debug|x64.ActiveCfg = Debug|Any CPU + {5016963A-6FC9-4063-AB83-2D1F9A2BC627}.Debug|x64.Build.0 = Debug|Any CPU + {5016963A-6FC9-4063-AB83-2D1F9A2BC627}.Debug|x86.ActiveCfg = Debug|Any CPU + {5016963A-6FC9-4063-AB83-2D1F9A2BC627}.Debug|x86.Build.0 = Debug|Any CPU + {5016963A-6FC9-4063-AB83-2D1F9A2BC627}.Release|Any CPU.ActiveCfg = Release|Any CPU + {5016963A-6FC9-4063-AB83-2D1F9A2BC627}.Release|Any CPU.Build.0 = Release|Any CPU + {5016963A-6FC9-4063-AB83-2D1F9A2BC627}.Release|x64.ActiveCfg = Release|Any CPU + {5016963A-6FC9-4063-AB83-2D1F9A2BC627}.Release|x64.Build.0 = Release|Any CPU + {5016963A-6FC9-4063-AB83-2D1F9A2BC627}.Release|x86.ActiveCfg = Release|Any CPU + {5016963A-6FC9-4063-AB83-2D1F9A2BC627}.Release|x86.Build.0 = Release|Any CPU + {72F43F43-F852-487F-8334-91D438CE2F7C}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {72F43F43-F852-487F-8334-91D438CE2F7C}.Debug|Any CPU.Build.0 = Debug|Any CPU + {72F43F43-F852-487F-8334-91D438CE2F7C}.Debug|x64.ActiveCfg = Debug|Any CPU + {72F43F43-F852-487F-8334-91D438CE2F7C}.Debug|x64.Build.0 = Debug|Any CPU + {72F43F43-F852-487F-8334-91D438CE2F7C}.Debug|x86.ActiveCfg = Debug|Any CPU + {72F43F43-F852-487F-8334-91D438CE2F7C}.Debug|x86.Build.0 = Debug|Any CPU + {72F43F43-F852-487F-8334-91D438CE2F7C}.Release|Any CPU.ActiveCfg = Release|Any CPU + {72F43F43-F852-487F-8334-91D438CE2F7C}.Release|Any CPU.Build.0 = Release|Any CPU + {72F43F43-F852-487F-8334-91D438CE2F7C}.Release|x64.ActiveCfg = Release|Any CPU + {72F43F43-F852-487F-8334-91D438CE2F7C}.Release|x64.Build.0 = Release|Any CPU + {72F43F43-F852-487F-8334-91D438CE2F7C}.Release|x86.ActiveCfg = Release|Any CPU + {72F43F43-F852-487F-8334-91D438CE2F7C}.Release|x86.Build.0 = Release|Any CPU + {A4DBF88F-34D0-4A05-ACCE-DE080F912FDB}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {A4DBF88F-34D0-4A05-ACCE-DE080F912FDB}.Debug|Any CPU.Build.0 = Debug|Any CPU + {A4DBF88F-34D0-4A05-ACCE-DE080F912FDB}.Debug|x64.ActiveCfg = Debug|Any CPU + {A4DBF88F-34D0-4A05-ACCE-DE080F912FDB}.Debug|x64.Build.0 = Debug|Any CPU + {A4DBF88F-34D0-4A05-ACCE-DE080F912FDB}.Debug|x86.ActiveCfg = Debug|Any CPU + {A4DBF88F-34D0-4A05-ACCE-DE080F912FDB}.Debug|x86.Build.0 = Debug|Any CPU + {A4DBF88F-34D0-4A05-ACCE-DE080F912FDB}.Release|Any CPU.ActiveCfg = Release|Any CPU + {A4DBF88F-34D0-4A05-ACCE-DE080F912FDB}.Release|Any CPU.Build.0 = Release|Any CPU + {A4DBF88F-34D0-4A05-ACCE-DE080F912FDB}.Release|x64.ActiveCfg = Release|Any CPU + {A4DBF88F-34D0-4A05-ACCE-DE080F912FDB}.Release|x64.Build.0 = Release|Any CPU + {A4DBF88F-34D0-4A05-ACCE-DE080F912FDB}.Release|x86.ActiveCfg = Release|Any CPU + {A4DBF88F-34D0-4A05-ACCE-DE080F912FDB}.Release|x86.Build.0 = Release|Any CPU + {F622D38D-DA49-473E-B724-E706F8113CF2}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {F622D38D-DA49-473E-B724-E706F8113CF2}.Debug|Any CPU.Build.0 = Debug|Any CPU + {F622D38D-DA49-473E-B724-E706F8113CF2}.Debug|x64.ActiveCfg = Debug|Any CPU + {F622D38D-DA49-473E-B724-E706F8113CF2}.Debug|x64.Build.0 = Debug|Any CPU + {F622D38D-DA49-473E-B724-E706F8113CF2}.Debug|x86.ActiveCfg = Debug|Any CPU + {F622D38D-DA49-473E-B724-E706F8113CF2}.Debug|x86.Build.0 = Debug|Any CPU + {F622D38D-DA49-473E-B724-E706F8113CF2}.Release|Any CPU.ActiveCfg = Release|Any CPU + {F622D38D-DA49-473E-B724-E706F8113CF2}.Release|Any CPU.Build.0 = Release|Any CPU + {F622D38D-DA49-473E-B724-E706F8113CF2}.Release|x64.ActiveCfg = Release|Any CPU + {F622D38D-DA49-473E-B724-E706F8113CF2}.Release|x64.Build.0 = Release|Any CPU + {F622D38D-DA49-473E-B724-E706F8113CF2}.Release|x86.ActiveCfg = Release|Any CPU + {F622D38D-DA49-473E-B724-E706F8113CF2}.Release|x86.Build.0 = Release|Any CPU + {3A3D7610-C864-4413-B07E-9E8C2A49A90E}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {3A3D7610-C864-4413-B07E-9E8C2A49A90E}.Debug|Any CPU.Build.0 = Debug|Any CPU + {3A3D7610-C864-4413-B07E-9E8C2A49A90E}.Debug|x64.ActiveCfg = Debug|Any CPU + {3A3D7610-C864-4413-B07E-9E8C2A49A90E}.Debug|x64.Build.0 = Debug|Any CPU + {3A3D7610-C864-4413-B07E-9E8C2A49A90E}.Debug|x86.ActiveCfg = Debug|Any CPU + {3A3D7610-C864-4413-B07E-9E8C2A49A90E}.Debug|x86.Build.0 = Debug|Any CPU + {3A3D7610-C864-4413-B07E-9E8C2A49A90E}.Release|Any CPU.ActiveCfg = Release|Any CPU + {3A3D7610-C864-4413-B07E-9E8C2A49A90E}.Release|Any CPU.Build.0 = Release|Any CPU + {3A3D7610-C864-4413-B07E-9E8C2A49A90E}.Release|x64.ActiveCfg = Release|Any CPU + {3A3D7610-C864-4413-B07E-9E8C2A49A90E}.Release|x64.Build.0 = Release|Any CPU + {3A3D7610-C864-4413-B07E-9E8C2A49A90E}.Release|x86.ActiveCfg = Release|Any CPU + {3A3D7610-C864-4413-B07E-9E8C2A49A90E}.Release|x86.Build.0 = Release|Any CPU + {9C4DEE96-CD7D-4AE3-A811-0B48B477003B}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {9C4DEE96-CD7D-4AE3-A811-0B48B477003B}.Debug|Any CPU.Build.0 = Debug|Any CPU + {9C4DEE96-CD7D-4AE3-A811-0B48B477003B}.Debug|x64.ActiveCfg = Debug|Any CPU + {9C4DEE96-CD7D-4AE3-A811-0B48B477003B}.Debug|x64.Build.0 = Debug|Any CPU + {9C4DEE96-CD7D-4AE3-A811-0B48B477003B}.Debug|x86.ActiveCfg = Debug|Any CPU + {9C4DEE96-CD7D-4AE3-A811-0B48B477003B}.Debug|x86.Build.0 = Debug|Any CPU + {9C4DEE96-CD7D-4AE3-A811-0B48B477003B}.Release|Any CPU.ActiveCfg = Release|Any CPU + {9C4DEE96-CD7D-4AE3-A811-0B48B477003B}.Release|Any CPU.Build.0 = Release|Any CPU + {9C4DEE96-CD7D-4AE3-A811-0B48B477003B}.Release|x64.ActiveCfg = Release|Any CPU + {9C4DEE96-CD7D-4AE3-A811-0B48B477003B}.Release|x64.Build.0 = Release|Any CPU + {9C4DEE96-CD7D-4AE3-A811-0B48B477003B}.Release|x86.ActiveCfg = Release|Any CPU + {9C4DEE96-CD7D-4AE3-A811-0B48B477003B}.Release|x86.Build.0 = Release|Any CPU + {437B2667-9461-47D2-B75B-4D2E03D69B94}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {437B2667-9461-47D2-B75B-4D2E03D69B94}.Debug|Any CPU.Build.0 = Debug|Any CPU + {437B2667-9461-47D2-B75B-4D2E03D69B94}.Debug|x64.ActiveCfg = Debug|Any CPU + {437B2667-9461-47D2-B75B-4D2E03D69B94}.Debug|x64.Build.0 = Debug|Any CPU + {437B2667-9461-47D2-B75B-4D2E03D69B94}.Debug|x86.ActiveCfg = Debug|Any CPU + {437B2667-9461-47D2-B75B-4D2E03D69B94}.Debug|x86.Build.0 = Debug|Any CPU + {437B2667-9461-47D2-B75B-4D2E03D69B94}.Release|Any CPU.ActiveCfg = Release|Any CPU + {437B2667-9461-47D2-B75B-4D2E03D69B94}.Release|Any CPU.Build.0 = Release|Any CPU + {437B2667-9461-47D2-B75B-4D2E03D69B94}.Release|x64.ActiveCfg = Release|Any CPU + {437B2667-9461-47D2-B75B-4D2E03D69B94}.Release|x64.Build.0 = Release|Any CPU + {437B2667-9461-47D2-B75B-4D2E03D69B94}.Release|x86.ActiveCfg = Release|Any CPU + {437B2667-9461-47D2-B75B-4D2E03D69B94}.Release|x86.Build.0 = Release|Any CPU + {8249DF28-CDAF-4DEF-A912-C27F57B67FD5}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {8249DF28-CDAF-4DEF-A912-C27F57B67FD5}.Debug|Any CPU.Build.0 = Debug|Any CPU + {8249DF28-CDAF-4DEF-A912-C27F57B67FD5}.Debug|x64.ActiveCfg = Debug|Any CPU + {8249DF28-CDAF-4DEF-A912-C27F57B67FD5}.Debug|x64.Build.0 = Debug|Any CPU + {8249DF28-CDAF-4DEF-A912-C27F57B67FD5}.Debug|x86.ActiveCfg = Debug|Any CPU + {8249DF28-CDAF-4DEF-A912-C27F57B67FD5}.Debug|x86.Build.0 = Debug|Any CPU + {8249DF28-CDAF-4DEF-A912-C27F57B67FD5}.Release|Any CPU.ActiveCfg = Release|Any CPU + {8249DF28-CDAF-4DEF-A912-C27F57B67FD5}.Release|Any CPU.Build.0 = Release|Any CPU + {8249DF28-CDAF-4DEF-A912-C27F57B67FD5}.Release|x64.ActiveCfg = Release|Any CPU + {8249DF28-CDAF-4DEF-A912-C27F57B67FD5}.Release|x64.Build.0 = Release|Any CPU + {8249DF28-CDAF-4DEF-A912-C27F57B67FD5}.Release|x86.ActiveCfg = Release|Any CPU + {8249DF28-CDAF-4DEF-A912-C27F57B67FD5}.Release|x86.Build.0 = Release|Any CPU + {CBFB015B-C069-475F-A476-D52222729804}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {CBFB015B-C069-475F-A476-D52222729804}.Debug|Any CPU.Build.0 = Debug|Any CPU + {CBFB015B-C069-475F-A476-D52222729804}.Debug|x64.ActiveCfg = Debug|Any CPU + {CBFB015B-C069-475F-A476-D52222729804}.Debug|x64.Build.0 = Debug|Any CPU + {CBFB015B-C069-475F-A476-D52222729804}.Debug|x86.ActiveCfg = Debug|Any CPU + {CBFB015B-C069-475F-A476-D52222729804}.Debug|x86.Build.0 = Debug|Any CPU + {CBFB015B-C069-475F-A476-D52222729804}.Release|Any CPU.ActiveCfg = Release|Any CPU + {CBFB015B-C069-475F-A476-D52222729804}.Release|Any CPU.Build.0 = Release|Any CPU + {CBFB015B-C069-475F-A476-D52222729804}.Release|x64.ActiveCfg = Release|Any CPU + {CBFB015B-C069-475F-A476-D52222729804}.Release|x64.Build.0 = Release|Any CPU + {CBFB015B-C069-475F-A476-D52222729804}.Release|x86.ActiveCfg = Release|Any CPU + {CBFB015B-C069-475F-A476-D52222729804}.Release|x86.Build.0 = Release|Any CPU + {2A41D9D2-3218-4F12-9C2B-3DB18A8E732E}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {2A41D9D2-3218-4F12-9C2B-3DB18A8E732E}.Debug|Any CPU.Build.0 = Debug|Any CPU + {2A41D9D2-3218-4F12-9C2B-3DB18A8E732E}.Debug|x64.ActiveCfg = Debug|Any CPU + {2A41D9D2-3218-4F12-9C2B-3DB18A8E732E}.Debug|x64.Build.0 = Debug|Any CPU + {2A41D9D2-3218-4F12-9C2B-3DB18A8E732E}.Debug|x86.ActiveCfg = Debug|Any CPU + {2A41D9D2-3218-4F12-9C2B-3DB18A8E732E}.Debug|x86.Build.0 = Debug|Any CPU + {2A41D9D2-3218-4F12-9C2B-3DB18A8E732E}.Release|Any CPU.ActiveCfg = Release|Any CPU + {2A41D9D2-3218-4F12-9C2B-3DB18A8E732E}.Release|Any CPU.Build.0 = Release|Any CPU + {2A41D9D2-3218-4F12-9C2B-3DB18A8E732E}.Release|x64.ActiveCfg = Release|Any CPU + {2A41D9D2-3218-4F12-9C2B-3DB18A8E732E}.Release|x64.Build.0 = Release|Any CPU + {2A41D9D2-3218-4F12-9C2B-3DB18A8E732E}.Release|x86.ActiveCfg = Release|Any CPU + {2A41D9D2-3218-4F12-9C2B-3DB18A8E732E}.Release|x86.Build.0 = Release|Any CPU + {3EB22234-642E-4533-BCC3-93E8ED443B1D}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {3EB22234-642E-4533-BCC3-93E8ED443B1D}.Debug|Any CPU.Build.0 = Debug|Any CPU + {3EB22234-642E-4533-BCC3-93E8ED443B1D}.Debug|x64.ActiveCfg = Debug|Any CPU + {3EB22234-642E-4533-BCC3-93E8ED443B1D}.Debug|x64.Build.0 = Debug|Any CPU + {3EB22234-642E-4533-BCC3-93E8ED443B1D}.Debug|x86.ActiveCfg = Debug|Any CPU + {3EB22234-642E-4533-BCC3-93E8ED443B1D}.Debug|x86.Build.0 = Debug|Any CPU + {3EB22234-642E-4533-BCC3-93E8ED443B1D}.Release|Any CPU.ActiveCfg = Release|Any CPU + {3EB22234-642E-4533-BCC3-93E8ED443B1D}.Release|Any CPU.Build.0 = Release|Any CPU + {3EB22234-642E-4533-BCC3-93E8ED443B1D}.Release|x64.ActiveCfg = Release|Any CPU + {3EB22234-642E-4533-BCC3-93E8ED443B1D}.Release|x64.Build.0 = Release|Any CPU + {3EB22234-642E-4533-BCC3-93E8ED443B1D}.Release|x86.ActiveCfg = Release|Any CPU + {3EB22234-642E-4533-BCC3-93E8ED443B1D}.Release|x86.Build.0 = Release|Any CPU + {84A5DE81-4444-499A-93BF-6DC4CA72F8D4}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {84A5DE81-4444-499A-93BF-6DC4CA72F8D4}.Debug|Any CPU.Build.0 = Debug|Any CPU + {84A5DE81-4444-499A-93BF-6DC4CA72F8D4}.Debug|x64.ActiveCfg = Debug|Any CPU + {84A5DE81-4444-499A-93BF-6DC4CA72F8D4}.Debug|x64.Build.0 = Debug|Any CPU + {84A5DE81-4444-499A-93BF-6DC4CA72F8D4}.Debug|x86.ActiveCfg = Debug|Any CPU + {84A5DE81-4444-499A-93BF-6DC4CA72F8D4}.Debug|x86.Build.0 = Debug|Any CPU + {84A5DE81-4444-499A-93BF-6DC4CA72F8D4}.Release|Any CPU.ActiveCfg = Release|Any CPU + {84A5DE81-4444-499A-93BF-6DC4CA72F8D4}.Release|Any CPU.Build.0 = Release|Any CPU + {84A5DE81-4444-499A-93BF-6DC4CA72F8D4}.Release|x64.ActiveCfg = Release|Any CPU + {84A5DE81-4444-499A-93BF-6DC4CA72F8D4}.Release|x64.Build.0 = Release|Any CPU + {84A5DE81-4444-499A-93BF-6DC4CA72F8D4}.Release|x86.ActiveCfg = Release|Any CPU + {84A5DE81-4444-499A-93BF-6DC4CA72F8D4}.Release|x86.Build.0 = Release|Any CPU + {42E21E1D-C3DE-4765-93E9-39391BB5C802}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {42E21E1D-C3DE-4765-93E9-39391BB5C802}.Debug|Any CPU.Build.0 = Debug|Any CPU + {42E21E1D-C3DE-4765-93E9-39391BB5C802}.Debug|x64.ActiveCfg = Debug|Any CPU + {42E21E1D-C3DE-4765-93E9-39391BB5C802}.Debug|x64.Build.0 = Debug|Any CPU + {42E21E1D-C3DE-4765-93E9-39391BB5C802}.Debug|x86.ActiveCfg = Debug|Any CPU + {42E21E1D-C3DE-4765-93E9-39391BB5C802}.Debug|x86.Build.0 = Debug|Any CPU + {42E21E1D-C3DE-4765-93E9-39391BB5C802}.Release|Any CPU.ActiveCfg = Release|Any CPU + {42E21E1D-C3DE-4765-93E9-39391BB5C802}.Release|Any CPU.Build.0 = Release|Any CPU + {42E21E1D-C3DE-4765-93E9-39391BB5C802}.Release|x64.ActiveCfg = Release|Any CPU + {42E21E1D-C3DE-4765-93E9-39391BB5C802}.Release|x64.Build.0 = Release|Any CPU + {42E21E1D-C3DE-4765-93E9-39391BB5C802}.Release|x86.ActiveCfg = Release|Any CPU + {42E21E1D-C3DE-4765-93E9-39391BB5C802}.Release|x86.Build.0 = Release|Any CPU + {B6E2EE26-B297-4AB9-A47E-A227F5EAE108}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {B6E2EE26-B297-4AB9-A47E-A227F5EAE108}.Debug|Any CPU.Build.0 = Debug|Any CPU + {B6E2EE26-B297-4AB9-A47E-A227F5EAE108}.Debug|x64.ActiveCfg = Debug|Any CPU + {B6E2EE26-B297-4AB9-A47E-A227F5EAE108}.Debug|x64.Build.0 = Debug|Any CPU + {B6E2EE26-B297-4AB9-A47E-A227F5EAE108}.Debug|x86.ActiveCfg = Debug|Any CPU + {B6E2EE26-B297-4AB9-A47E-A227F5EAE108}.Debug|x86.Build.0 = Debug|Any CPU + {B6E2EE26-B297-4AB9-A47E-A227F5EAE108}.Release|Any CPU.ActiveCfg = Release|Any CPU + {B6E2EE26-B297-4AB9-A47E-A227F5EAE108}.Release|Any CPU.Build.0 = Release|Any CPU + {B6E2EE26-B297-4AB9-A47E-A227F5EAE108}.Release|x64.ActiveCfg = Release|Any CPU + {B6E2EE26-B297-4AB9-A47E-A227F5EAE108}.Release|x64.Build.0 = Release|Any CPU + {B6E2EE26-B297-4AB9-A47E-A227F5EAE108}.Release|x86.ActiveCfg = Release|Any CPU + {B6E2EE26-B297-4AB9-A47E-A227F5EAE108}.Release|x86.Build.0 = Release|Any CPU + {CDB2D636-C82F-43F1-BB30-FFC6258FBAB4}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {CDB2D636-C82F-43F1-BB30-FFC6258FBAB4}.Debug|Any CPU.Build.0 = Debug|Any CPU + {CDB2D636-C82F-43F1-BB30-FFC6258FBAB4}.Debug|x64.ActiveCfg = Debug|Any CPU + {CDB2D636-C82F-43F1-BB30-FFC6258FBAB4}.Debug|x64.Build.0 = Debug|Any CPU + {CDB2D636-C82F-43F1-BB30-FFC6258FBAB4}.Debug|x86.ActiveCfg = Debug|Any CPU + {CDB2D636-C82F-43F1-BB30-FFC6258FBAB4}.Debug|x86.Build.0 = Debug|Any CPU + {CDB2D636-C82F-43F1-BB30-FFC6258FBAB4}.Release|Any CPU.ActiveCfg = Release|Any CPU + {CDB2D636-C82F-43F1-BB30-FFC6258FBAB4}.Release|Any CPU.Build.0 = Release|Any CPU + {CDB2D636-C82F-43F1-BB30-FFC6258FBAB4}.Release|x64.ActiveCfg = Release|Any CPU + {CDB2D636-C82F-43F1-BB30-FFC6258FBAB4}.Release|x64.Build.0 = Release|Any CPU + {CDB2D636-C82F-43F1-BB30-FFC6258FBAB4}.Release|x86.ActiveCfg = Release|Any CPU + {CDB2D636-C82F-43F1-BB30-FFC6258FBAB4}.Release|x86.Build.0 = Release|Any CPU + {2891FCDE-BB89-46F0-A40C-368EF804DB44}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {2891FCDE-BB89-46F0-A40C-368EF804DB44}.Debug|Any CPU.Build.0 = Debug|Any CPU + {2891FCDE-BB89-46F0-A40C-368EF804DB44}.Debug|x64.ActiveCfg = Debug|Any CPU + {2891FCDE-BB89-46F0-A40C-368EF804DB44}.Debug|x64.Build.0 = Debug|Any CPU + {2891FCDE-BB89-46F0-A40C-368EF804DB44}.Debug|x86.ActiveCfg = Debug|Any CPU + {2891FCDE-BB89-46F0-A40C-368EF804DB44}.Debug|x86.Build.0 = Debug|Any CPU + {2891FCDE-BB89-46F0-A40C-368EF804DB44}.Release|Any CPU.ActiveCfg = Release|Any CPU + {2891FCDE-BB89-46F0-A40C-368EF804DB44}.Release|Any CPU.Build.0 = Release|Any CPU + {2891FCDE-BB89-46F0-A40C-368EF804DB44}.Release|x64.ActiveCfg = Release|Any CPU + {2891FCDE-BB89-46F0-A40C-368EF804DB44}.Release|x64.Build.0 = Release|Any CPU + {2891FCDE-BB89-46F0-A40C-368EF804DB44}.Release|x86.ActiveCfg = Release|Any CPU + {2891FCDE-BB89-46F0-A40C-368EF804DB44}.Release|x86.Build.0 = Release|Any CPU + {B91C60FB-926F-47C3-BFD0-6DD145308344}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {B91C60FB-926F-47C3-BFD0-6DD145308344}.Debug|Any CPU.Build.0 = Debug|Any CPU + {B91C60FB-926F-47C3-BFD0-6DD145308344}.Debug|x64.ActiveCfg = Debug|Any CPU + {B91C60FB-926F-47C3-BFD0-6DD145308344}.Debug|x64.Build.0 = Debug|Any CPU + {B91C60FB-926F-47C3-BFD0-6DD145308344}.Debug|x86.ActiveCfg = Debug|Any CPU + {B91C60FB-926F-47C3-BFD0-6DD145308344}.Debug|x86.Build.0 = Debug|Any CPU + {B91C60FB-926F-47C3-BFD0-6DD145308344}.Release|Any CPU.ActiveCfg = Release|Any CPU + {B91C60FB-926F-47C3-BFD0-6DD145308344}.Release|Any CPU.Build.0 = Release|Any CPU + {B91C60FB-926F-47C3-BFD0-6DD145308344}.Release|x64.ActiveCfg = Release|Any CPU + {B91C60FB-926F-47C3-BFD0-6DD145308344}.Release|x64.Build.0 = Release|Any CPU + {B91C60FB-926F-47C3-BFD0-6DD145308344}.Release|x86.ActiveCfg = Release|Any CPU + {B91C60FB-926F-47C3-BFD0-6DD145308344}.Release|x86.Build.0 = Release|Any CPU + {30DF89D1-D66D-4078-8A3B-951637A42265}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {30DF89D1-D66D-4078-8A3B-951637A42265}.Debug|Any CPU.Build.0 = Debug|Any CPU + {30DF89D1-D66D-4078-8A3B-951637A42265}.Debug|x64.ActiveCfg = Debug|Any CPU + {30DF89D1-D66D-4078-8A3B-951637A42265}.Debug|x64.Build.0 = Debug|Any CPU + {30DF89D1-D66D-4078-8A3B-951637A42265}.Debug|x86.ActiveCfg = Debug|Any CPU + {30DF89D1-D66D-4078-8A3B-951637A42265}.Debug|x86.Build.0 = Debug|Any CPU + {30DF89D1-D66D-4078-8A3B-951637A42265}.Release|Any CPU.ActiveCfg = Release|Any CPU + {30DF89D1-D66D-4078-8A3B-951637A42265}.Release|Any CPU.Build.0 = Release|Any CPU + {30DF89D1-D66D-4078-8A3B-951637A42265}.Release|x64.ActiveCfg = Release|Any CPU + {30DF89D1-D66D-4078-8A3B-951637A42265}.Release|x64.Build.0 = Release|Any CPU + {30DF89D1-D66D-4078-8A3B-951637A42265}.Release|x86.ActiveCfg = Release|Any CPU + {30DF89D1-D66D-4078-8A3B-951637A42265}.Release|x86.Build.0 = Release|Any CPU + {6E98C770-72FF-41FA-8C42-30AABAAF5B4E}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {6E98C770-72FF-41FA-8C42-30AABAAF5B4E}.Debug|Any CPU.Build.0 = Debug|Any CPU + {6E98C770-72FF-41FA-8C42-30AABAAF5B4E}.Debug|x64.ActiveCfg = Debug|Any CPU + {6E98C770-72FF-41FA-8C42-30AABAAF5B4E}.Debug|x64.Build.0 = Debug|Any CPU + {6E98C770-72FF-41FA-8C42-30AABAAF5B4E}.Debug|x86.ActiveCfg = Debug|Any CPU + {6E98C770-72FF-41FA-8C42-30AABAAF5B4E}.Debug|x86.Build.0 = Debug|Any CPU + {6E98C770-72FF-41FA-8C42-30AABAAF5B4E}.Release|Any CPU.ActiveCfg = Release|Any CPU + {6E98C770-72FF-41FA-8C42-30AABAAF5B4E}.Release|Any CPU.Build.0 = Release|Any CPU + {6E98C770-72FF-41FA-8C42-30AABAAF5B4E}.Release|x64.ActiveCfg = Release|Any CPU + {6E98C770-72FF-41FA-8C42-30AABAAF5B4E}.Release|x64.Build.0 = Release|Any CPU + {6E98C770-72FF-41FA-8C42-30AABAAF5B4E}.Release|x86.ActiveCfg = Release|Any CPU + {6E98C770-72FF-41FA-8C42-30AABAAF5B4E}.Release|x86.Build.0 = Release|Any CPU + {79B36C92-BA93-4406-AB75-6F2282DDFF01}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {79B36C92-BA93-4406-AB75-6F2282DDFF01}.Debug|Any CPU.Build.0 = Debug|Any CPU + {79B36C92-BA93-4406-AB75-6F2282DDFF01}.Debug|x64.ActiveCfg = Debug|Any CPU + {79B36C92-BA93-4406-AB75-6F2282DDFF01}.Debug|x64.Build.0 = Debug|Any CPU + {79B36C92-BA93-4406-AB75-6F2282DDFF01}.Debug|x86.ActiveCfg = Debug|Any CPU + {79B36C92-BA93-4406-AB75-6F2282DDFF01}.Debug|x86.Build.0 = Debug|Any CPU + {79B36C92-BA93-4406-AB75-6F2282DDFF01}.Release|Any CPU.ActiveCfg = Release|Any CPU + {79B36C92-BA93-4406-AB75-6F2282DDFF01}.Release|Any CPU.Build.0 = Release|Any CPU + {79B36C92-BA93-4406-AB75-6F2282DDFF01}.Release|x64.ActiveCfg = Release|Any CPU + {79B36C92-BA93-4406-AB75-6F2282DDFF01}.Release|x64.Build.0 = Release|Any CPU + {79B36C92-BA93-4406-AB75-6F2282DDFF01}.Release|x86.ActiveCfg = Release|Any CPU + {79B36C92-BA93-4406-AB75-6F2282DDFF01}.Release|x86.Build.0 = Release|Any CPU + {4B60FA53-81F6-4AB6-BE9F-DE0992E11977}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {4B60FA53-81F6-4AB6-BE9F-DE0992E11977}.Debug|Any CPU.Build.0 = Debug|Any CPU + {4B60FA53-81F6-4AB6-BE9F-DE0992E11977}.Debug|x64.ActiveCfg = Debug|Any CPU + {4B60FA53-81F6-4AB6-BE9F-DE0992E11977}.Debug|x64.Build.0 = Debug|Any CPU + {4B60FA53-81F6-4AB6-BE9F-DE0992E11977}.Debug|x86.ActiveCfg = Debug|Any CPU + {4B60FA53-81F6-4AB6-BE9F-DE0992E11977}.Debug|x86.Build.0 = Debug|Any CPU + {4B60FA53-81F6-4AB6-BE9F-DE0992E11977}.Release|Any CPU.ActiveCfg = Release|Any CPU + {4B60FA53-81F6-4AB6-BE9F-DE0992E11977}.Release|Any CPU.Build.0 = Release|Any CPU + {4B60FA53-81F6-4AB6-BE9F-DE0992E11977}.Release|x64.ActiveCfg = Release|Any CPU + {4B60FA53-81F6-4AB6-BE9F-DE0992E11977}.Release|x64.Build.0 = Release|Any CPU + {4B60FA53-81F6-4AB6-BE9F-DE0992E11977}.Release|x86.ActiveCfg = Release|Any CPU + {4B60FA53-81F6-4AB6-BE9F-DE0992E11977}.Release|x86.Build.0 = Release|Any CPU + {6BBA820B-8443-4832-91C3-3AB002006494}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {6BBA820B-8443-4832-91C3-3AB002006494}.Debug|Any CPU.Build.0 = Debug|Any CPU + {6BBA820B-8443-4832-91C3-3AB002006494}.Debug|x64.ActiveCfg = Debug|Any CPU + {6BBA820B-8443-4832-91C3-3AB002006494}.Debug|x64.Build.0 = Debug|Any CPU + {6BBA820B-8443-4832-91C3-3AB002006494}.Debug|x86.ActiveCfg = Debug|Any CPU + {6BBA820B-8443-4832-91C3-3AB002006494}.Debug|x86.Build.0 = Debug|Any CPU + {6BBA820B-8443-4832-91C3-3AB002006494}.Release|Any CPU.ActiveCfg = Release|Any CPU + {6BBA820B-8443-4832-91C3-3AB002006494}.Release|Any CPU.Build.0 = Release|Any CPU + {6BBA820B-8443-4832-91C3-3AB002006494}.Release|x64.ActiveCfg = Release|Any CPU + {6BBA820B-8443-4832-91C3-3AB002006494}.Release|x64.Build.0 = Release|Any CPU + {6BBA820B-8443-4832-91C3-3AB002006494}.Release|x86.ActiveCfg = Release|Any CPU + {6BBA820B-8443-4832-91C3-3AB002006494}.Release|x86.Build.0 = Release|Any CPU + {7845AE1C-FBD7-4177-A06F-D7AAE8315DB2}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {7845AE1C-FBD7-4177-A06F-D7AAE8315DB2}.Debug|Any CPU.Build.0 = Debug|Any CPU + {7845AE1C-FBD7-4177-A06F-D7AAE8315DB2}.Debug|x64.ActiveCfg = Debug|Any CPU + {7845AE1C-FBD7-4177-A06F-D7AAE8315DB2}.Debug|x64.Build.0 = Debug|Any CPU + {7845AE1C-FBD7-4177-A06F-D7AAE8315DB2}.Debug|x86.ActiveCfg = Debug|Any CPU + {7845AE1C-FBD7-4177-A06F-D7AAE8315DB2}.Debug|x86.Build.0 = Debug|Any CPU + {7845AE1C-FBD7-4177-A06F-D7AAE8315DB2}.Release|Any CPU.ActiveCfg = Release|Any CPU + {7845AE1C-FBD7-4177-A06F-D7AAE8315DB2}.Release|Any CPU.Build.0 = Release|Any CPU + {7845AE1C-FBD7-4177-A06F-D7AAE8315DB2}.Release|x64.ActiveCfg = Release|Any CPU + {7845AE1C-FBD7-4177-A06F-D7AAE8315DB2}.Release|x64.Build.0 = Release|Any CPU + {7845AE1C-FBD7-4177-A06F-D7AAE8315DB2}.Release|x86.ActiveCfg = Release|Any CPU + {7845AE1C-FBD7-4177-A06F-D7AAE8315DB2}.Release|x86.Build.0 = Release|Any CPU + {F892BFFD-9101-4D59-B6FD-C532EB04D51F}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {F892BFFD-9101-4D59-B6FD-C532EB04D51F}.Debug|Any CPU.Build.0 = Debug|Any CPU + {F892BFFD-9101-4D59-B6FD-C532EB04D51F}.Debug|x64.ActiveCfg = Debug|Any CPU + {F892BFFD-9101-4D59-B6FD-C532EB04D51F}.Debug|x64.Build.0 = Debug|Any CPU + {F892BFFD-9101-4D59-B6FD-C532EB04D51F}.Debug|x86.ActiveCfg = Debug|Any CPU + {F892BFFD-9101-4D59-B6FD-C532EB04D51F}.Debug|x86.Build.0 = Debug|Any CPU + {F892BFFD-9101-4D59-B6FD-C532EB04D51F}.Release|Any CPU.ActiveCfg = Release|Any CPU + {F892BFFD-9101-4D59-B6FD-C532EB04D51F}.Release|Any CPU.Build.0 = Release|Any CPU + {F892BFFD-9101-4D59-B6FD-C532EB04D51F}.Release|x64.ActiveCfg = Release|Any CPU + {F892BFFD-9101-4D59-B6FD-C532EB04D51F}.Release|x64.Build.0 = Release|Any CPU + {F892BFFD-9101-4D59-B6FD-C532EB04D51F}.Release|x86.ActiveCfg = Release|Any CPU + {F892BFFD-9101-4D59-B6FD-C532EB04D51F}.Release|x86.Build.0 = Release|Any CPU + {EAE910FC-188C-41C3-822A-623964CABE48}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {EAE910FC-188C-41C3-822A-623964CABE48}.Debug|Any CPU.Build.0 = Debug|Any CPU + {EAE910FC-188C-41C3-822A-623964CABE48}.Debug|x64.ActiveCfg = Debug|Any CPU + {EAE910FC-188C-41C3-822A-623964CABE48}.Debug|x64.Build.0 = Debug|Any CPU + {EAE910FC-188C-41C3-822A-623964CABE48}.Debug|x86.ActiveCfg = Debug|Any CPU + {EAE910FC-188C-41C3-822A-623964CABE48}.Debug|x86.Build.0 = Debug|Any CPU + {EAE910FC-188C-41C3-822A-623964CABE48}.Release|Any CPU.ActiveCfg = Release|Any CPU + {EAE910FC-188C-41C3-822A-623964CABE48}.Release|Any CPU.Build.0 = Release|Any CPU + {EAE910FC-188C-41C3-822A-623964CABE48}.Release|x64.ActiveCfg = Release|Any CPU + {EAE910FC-188C-41C3-822A-623964CABE48}.Release|x64.Build.0 = Release|Any CPU + {EAE910FC-188C-41C3-822A-623964CABE48}.Release|x86.ActiveCfg = Release|Any CPU + {EAE910FC-188C-41C3-822A-623964CABE48}.Release|x86.Build.0 = Release|Any CPU + {BBA5C780-6348-427D-9600-726EAA8963B3}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {BBA5C780-6348-427D-9600-726EAA8963B3}.Debug|Any CPU.Build.0 = Debug|Any CPU + {BBA5C780-6348-427D-9600-726EAA8963B3}.Debug|x64.ActiveCfg = Debug|Any CPU + {BBA5C780-6348-427D-9600-726EAA8963B3}.Debug|x64.Build.0 = Debug|Any CPU + {BBA5C780-6348-427D-9600-726EAA8963B3}.Debug|x86.ActiveCfg = Debug|Any CPU + {BBA5C780-6348-427D-9600-726EAA8963B3}.Debug|x86.Build.0 = Debug|Any CPU + {BBA5C780-6348-427D-9600-726EAA8963B3}.Release|Any CPU.ActiveCfg = Release|Any CPU + {BBA5C780-6348-427D-9600-726EAA8963B3}.Release|Any CPU.Build.0 = Release|Any CPU + {BBA5C780-6348-427D-9600-726EAA8963B3}.Release|x64.ActiveCfg = Release|Any CPU + {BBA5C780-6348-427D-9600-726EAA8963B3}.Release|x64.Build.0 = Release|Any CPU + {BBA5C780-6348-427D-9600-726EAA8963B3}.Release|x86.ActiveCfg = Release|Any CPU + {BBA5C780-6348-427D-9600-726EAA8963B3}.Release|x86.Build.0 = Release|Any CPU + {5F44A429-816A-4560-A5AA-61CD23FD8A19}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {5F44A429-816A-4560-A5AA-61CD23FD8A19}.Debug|Any CPU.Build.0 = Debug|Any CPU + {5F44A429-816A-4560-A5AA-61CD23FD8A19}.Debug|x64.ActiveCfg = Debug|Any CPU + {5F44A429-816A-4560-A5AA-61CD23FD8A19}.Debug|x64.Build.0 = Debug|Any CPU + {5F44A429-816A-4560-A5AA-61CD23FD8A19}.Debug|x86.ActiveCfg = Debug|Any CPU + {5F44A429-816A-4560-A5AA-61CD23FD8A19}.Debug|x86.Build.0 = Debug|Any CPU + {5F44A429-816A-4560-A5AA-61CD23FD8A19}.Release|Any CPU.ActiveCfg = Release|Any CPU + {5F44A429-816A-4560-A5AA-61CD23FD8A19}.Release|Any CPU.Build.0 = Release|Any CPU + {5F44A429-816A-4560-A5AA-61CD23FD8A19}.Release|x64.ActiveCfg = Release|Any CPU + {5F44A429-816A-4560-A5AA-61CD23FD8A19}.Release|x64.Build.0 = Release|Any CPU + {5F44A429-816A-4560-A5AA-61CD23FD8A19}.Release|x86.ActiveCfg = Release|Any CPU + {5F44A429-816A-4560-A5AA-61CD23FD8A19}.Release|x86.Build.0 = Release|Any CPU + {20FDC3B4-9908-4ABF-BA1D-50E0B4A64F4B}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {20FDC3B4-9908-4ABF-BA1D-50E0B4A64F4B}.Debug|Any CPU.Build.0 = Debug|Any CPU + {20FDC3B4-9908-4ABF-BA1D-50E0B4A64F4B}.Debug|x64.ActiveCfg = Debug|Any CPU + {20FDC3B4-9908-4ABF-BA1D-50E0B4A64F4B}.Debug|x64.Build.0 = Debug|Any CPU + {20FDC3B4-9908-4ABF-BA1D-50E0B4A64F4B}.Debug|x86.ActiveCfg = Debug|Any CPU + {20FDC3B4-9908-4ABF-BA1D-50E0B4A64F4B}.Debug|x86.Build.0 = Debug|Any CPU + {20FDC3B4-9908-4ABF-BA1D-50E0B4A64F4B}.Release|Any CPU.ActiveCfg = Release|Any CPU + {20FDC3B4-9908-4ABF-BA1D-50E0B4A64F4B}.Release|Any CPU.Build.0 = Release|Any CPU + {20FDC3B4-9908-4ABF-BA1D-50E0B4A64F4B}.Release|x64.ActiveCfg = Release|Any CPU + {20FDC3B4-9908-4ABF-BA1D-50E0B4A64F4B}.Release|x64.Build.0 = Release|Any CPU + {20FDC3B4-9908-4ABF-BA1D-50E0B4A64F4B}.Release|x86.ActiveCfg = Release|Any CPU + {20FDC3B4-9908-4ABF-BA1D-50E0B4A64F4B}.Release|x86.Build.0 = Release|Any CPU + {544DBB82-4639-4856-A5F2-76828F7A8396}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {544DBB82-4639-4856-A5F2-76828F7A8396}.Debug|Any CPU.Build.0 = Debug|Any CPU + {544DBB82-4639-4856-A5F2-76828F7A8396}.Debug|x64.ActiveCfg = Debug|Any CPU + {544DBB82-4639-4856-A5F2-76828F7A8396}.Debug|x64.Build.0 = Debug|Any CPU + {544DBB82-4639-4856-A5F2-76828F7A8396}.Debug|x86.ActiveCfg = Debug|Any CPU + {544DBB82-4639-4856-A5F2-76828F7A8396}.Debug|x86.Build.0 = Debug|Any CPU + {544DBB82-4639-4856-A5F2-76828F7A8396}.Release|Any CPU.ActiveCfg = Release|Any CPU + {544DBB82-4639-4856-A5F2-76828F7A8396}.Release|Any CPU.Build.0 = Release|Any CPU + {544DBB82-4639-4856-A5F2-76828F7A8396}.Release|x64.ActiveCfg = Release|Any CPU + {544DBB82-4639-4856-A5F2-76828F7A8396}.Release|x64.Build.0 = Release|Any CPU + {544DBB82-4639-4856-A5F2-76828F7A8396}.Release|x86.ActiveCfg = Release|Any CPU + {544DBB82-4639-4856-A5F2-76828F7A8396}.Release|x86.Build.0 = Release|Any CPU + EndGlobalSection + GlobalSection(SolutionProperties) = preSolution + HideSolutionNode = FALSE + EndGlobalSection +EndGlobal diff --git a/src/StellaOps.Plugin/DependencyInjection/PluginDependencyInjectionExtensions.cs b/src/StellaOps.Plugin/DependencyInjection/PluginDependencyInjectionExtensions.cs new file mode 100644 index 00000000..40d72992 --- /dev/null +++ b/src/StellaOps.Plugin/DependencyInjection/PluginDependencyInjectionExtensions.cs @@ -0,0 +1,91 @@ +using Microsoft.Extensions.Configuration; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Logging; +using StellaOps.DependencyInjection; +using StellaOps.Plugin.Hosting; +using StellaOps.Plugin.Internal; +using System; +using System.Collections.Generic; +using System.Linq; + +namespace StellaOps.Plugin.DependencyInjection; + +public static class PluginDependencyInjectionExtensions +{ + public static IServiceCollection RegisterPluginRoutines( + this IServiceCollection services, + IConfiguration configuration, + PluginHostOptions options, + ILogger? logger = null) + { + if (services == null) + { + throw new ArgumentNullException(nameof(services)); + } + + if (configuration == null) + { + throw new ArgumentNullException(nameof(configuration)); + } + + if (options == null) + { + throw new ArgumentNullException(nameof(options)); + } + + var loadResult = PluginHost.LoadPlugins(options, logger); + + foreach (var plugin in loadResult.Plugins) + { + foreach (var routine in CreateRoutines(plugin.Assembly)) + { + logger?.LogDebug( + "Registering DI routine '{RoutineType}' from plugin '{PluginAssembly}'.", + routine.GetType().FullName, + plugin.Assembly.FullName); + + routine.Register(services, configuration); + } + } + + if (loadResult.MissingOrderedPlugins.Count > 0) + { + logger?.LogWarning( + "Some ordered plugins were not found: {Missing}", + string.Join(", ", loadResult.MissingOrderedPlugins)); + } + + return services; + } + + private static IEnumerable CreateRoutines(System.Reflection.Assembly assembly) + { + foreach (var type in assembly.GetLoadableTypes()) + { + if (type is null || type.IsAbstract || type.IsInterface) + { + continue; + } + + if (!typeof(IDependencyInjectionRoutine).IsAssignableFrom(type)) + { + continue; + } + + object? instance; + try + { + instance = Activator.CreateInstance(type); + } + catch + { + continue; + } + + if (instance is IDependencyInjectionRoutine routine) + { + yield return routine; + } + } + } +} \ No newline at end of file diff --git a/src/StellaOps.Plugin/DependencyInjection/StellaOpsPluginRegistration.cs b/src/StellaOps.Plugin/DependencyInjection/StellaOpsPluginRegistration.cs new file mode 100644 index 00000000..c228b2da --- /dev/null +++ b/src/StellaOps.Plugin/DependencyInjection/StellaOpsPluginRegistration.cs @@ -0,0 +1,26 @@ +using Microsoft.Extensions.Configuration; +using Microsoft.Extensions.DependencyInjection; +using StellaOps.DependencyInjection; + +namespace StellaOps.Plugin.DependencyInjection; + +public static class StellaOpsPluginRegistration +{ + public static IServiceCollection RegisterStellaOpsPlugin( + this IServiceCollection services, + IConfiguration configuration) + { + // No-op today but reserved for future plugin infrastructure services. + return services; + } +} + +public sealed class DependencyInjectionRoutine : IDependencyInjectionRoutine +{ + public IServiceCollection Register( + IServiceCollection services, + IConfiguration configuration) + { + return services.RegisterStellaOpsPlugin(configuration); + } +} \ No newline at end of file diff --git a/src/StellaOps.Plugin/Hosting/PluginAssembly.cs b/src/StellaOps.Plugin/Hosting/PluginAssembly.cs new file mode 100644 index 00000000..12f31d0a --- /dev/null +++ b/src/StellaOps.Plugin/Hosting/PluginAssembly.cs @@ -0,0 +1,21 @@ +using System.Reflection; + +namespace StellaOps.Plugin.Hosting; + +public sealed class PluginAssembly +{ + internal PluginAssembly(string assemblyPath, Assembly assembly, PluginLoadContext loadContext) + { + AssemblyPath = assemblyPath; + Assembly = assembly; + LoadContext = loadContext; + } + + public string AssemblyPath { get; } + + public Assembly Assembly { get; } + + internal PluginLoadContext LoadContext { get; } + + public override string ToString() => Assembly.FullName ?? AssemblyPath; +} \ No newline at end of file diff --git a/src/StellaOps.Plugin/Hosting/PluginHost.cs b/src/StellaOps.Plugin/Hosting/PluginHost.cs new file mode 100644 index 00000000..b37a1c03 --- /dev/null +++ b/src/StellaOps.Plugin/Hosting/PluginHost.cs @@ -0,0 +1,216 @@ +using Microsoft.Extensions.Logging; +using System; +using System.Collections.Generic; +using System.Collections.ObjectModel; +using System.IO; +using System.Linq; + +namespace StellaOps.Plugin.Hosting; + +public static class PluginHost +{ + private static readonly object Sync = new(); + private static readonly Dictionary LoadedPlugins = new(StringComparer.OrdinalIgnoreCase); + + public static PluginHostResult LoadPlugins(PluginHostOptions options, ILogger? logger = null) + { + if (options == null) + { + throw new ArgumentNullException(nameof(options)); + } + + var baseDirectory = options.ResolveBaseDirectory(); + var pluginDirectory = ResolvePluginDirectory(options, baseDirectory); + + if (options.EnsureDirectoryExists && !Directory.Exists(pluginDirectory)) + { + Directory.CreateDirectory(pluginDirectory); + } + + if (!Directory.Exists(pluginDirectory)) + { + logger?.LogWarning("Plugin directory '{PluginDirectory}' does not exist; no plugins will be loaded.", pluginDirectory); + return new PluginHostResult(pluginDirectory, Array.Empty(), Array.Empty(), Array.Empty()); + } + + var searchPatterns = BuildSearchPatterns(options, pluginDirectory); + var discovered = DiscoverPluginFiles(pluginDirectory, searchPatterns, options.RecursiveSearch, logger); + var orderedFiles = ApplyExplicitOrdering(discovered, options.PluginOrder, out var missingOrderedNames); + + var loaded = new List(orderedFiles.Count); + + lock (Sync) + { + foreach (var file in orderedFiles) + { + if (LoadedPlugins.TryGetValue(file, out var existing)) + { + loaded.Add(existing); + continue; + } + + try + { + var loadContext = new PluginLoadContext(file); + var assembly = loadContext.LoadFromAssemblyPath(file); + var descriptor = new PluginAssembly(file, assembly, loadContext); + LoadedPlugins[file] = descriptor; + loaded.Add(descriptor); + logger?.LogInformation("Loaded plugin assembly '{Assembly}' from '{Path}'.", assembly.FullName, file); + } + catch (Exception ex) + { + logger?.LogError(ex, "Failed to load plugin assembly from '{Path}'.", file); + } + } + } + + var missingOrdered = new ReadOnlyCollection(missingOrderedNames); + return new PluginHostResult(pluginDirectory, searchPatterns, new ReadOnlyCollection(loaded), missingOrdered); + } + + private static string ResolvePluginDirectory(PluginHostOptions options, string baseDirectory) + { + if (string.IsNullOrWhiteSpace(options.PluginsDirectory)) + { + return Path.Combine(baseDirectory, "PluginBinaries"); + } + + if (Path.IsPathRooted(options.PluginsDirectory)) + { + return options.PluginsDirectory; + } + + return Path.Combine(baseDirectory, options.PluginsDirectory); + } + + private static IReadOnlyList BuildSearchPatterns(PluginHostOptions options, string pluginDirectory) + { + var patterns = new List(); + if (options.SearchPatterns.Count > 0) + { + patterns.AddRange(options.SearchPatterns); + } + else + { + var prefixes = new List(); + if (!string.IsNullOrWhiteSpace(options.PrimaryPrefix)) + { + prefixes.Add(options.PrimaryPrefix); + } + else if (System.Reflection.Assembly.GetEntryAssembly()?.GetName().Name is { } entryName) + { + prefixes.Add(entryName); + } + + prefixes.AddRange(options.AdditionalPrefixes); + + if (prefixes.Count == 0) + { + // Fallback to directory name + prefixes.Add(Path.GetFileName(pluginDirectory)); + } + + foreach (var prefix in prefixes.Where(p => !string.IsNullOrWhiteSpace(p))) + { + patterns.Add($"{prefix}.Plugin.*.dll"); + } + } + + return new ReadOnlyCollection(patterns.Distinct(StringComparer.OrdinalIgnoreCase).ToList()); + } + + private static List DiscoverPluginFiles( + string pluginDirectory, + IReadOnlyList searchPatterns, + bool recurse, + ILogger? logger) + { + var files = new List(); + var seen = new HashSet(StringComparer.OrdinalIgnoreCase); + var searchOption = recurse ? SearchOption.AllDirectories : SearchOption.TopDirectoryOnly; + + foreach (var pattern in searchPatterns) + { + try + { + foreach (var file in Directory.EnumerateFiles(pluginDirectory, pattern, searchOption)) + { + if (IsHiddenPath(file)) + { + continue; + } + + if (seen.Add(file)) + { + files.Add(file); + } + } + } + catch (DirectoryNotFoundException) + { + // Directory could be removed between the existence check and enumeration. + logger?.LogDebug("Plugin directory '{PluginDirectory}' disappeared before enumeration.", pluginDirectory); + } + } + + return files; + } + + private static List ApplyExplicitOrdering( + List discoveredFiles, + IList pluginOrder, + out List missingNames) + { + if (pluginOrder.Count == 0 || discoveredFiles.Count == 0) + { + missingNames = new List(); + discoveredFiles.Sort(StringComparer.OrdinalIgnoreCase); + return discoveredFiles; + } + + var configuredSet = new HashSet(pluginOrder, StringComparer.OrdinalIgnoreCase); + var fileLookup = discoveredFiles.ToDictionary( + k => Path.GetFileNameWithoutExtension(k), + StringComparer.OrdinalIgnoreCase); + + var specified = new List(); + foreach (var name in pluginOrder) + { + if (fileLookup.TryGetValue(name, out var file)) + { + specified.Add(file); + } + } + + var unspecified = discoveredFiles + .Where(f => !configuredSet.Contains(Path.GetFileNameWithoutExtension(f))) + .OrderBy(f => f, StringComparer.OrdinalIgnoreCase) + .ToList(); + + missingNames = pluginOrder + .Where(name => !fileLookup.ContainsKey(name)) + .Distinct(StringComparer.OrdinalIgnoreCase) + .ToList(); + + specified.AddRange(unspecified); + return specified; + } + + private static bool IsHiddenPath(string filePath) + { + var directory = Path.GetDirectoryName(filePath); + while (!string.IsNullOrEmpty(directory)) + { + var name = Path.GetFileName(directory); + if (name.StartsWith(".", StringComparison.Ordinal)) + { + return true; + } + + directory = Path.GetDirectoryName(directory); + } + + return false; + } +} \ No newline at end of file diff --git a/src/StellaOps.Plugin/Hosting/PluginHostOptions.cs b/src/StellaOps.Plugin/Hosting/PluginHostOptions.cs new file mode 100644 index 00000000..634a2f2e --- /dev/null +++ b/src/StellaOps.Plugin/Hosting/PluginHostOptions.cs @@ -0,0 +1,59 @@ +using System; +using System.Collections.Generic; +using System.IO; + +namespace StellaOps.Plugin.Hosting; + +public sealed class PluginHostOptions +{ + private readonly List additionalPrefixes = new(); + private readonly List pluginOrder = new(); + private readonly List searchPatterns = new(); + + /// + /// Optional base directory used for resolving relative plugin paths. Defaults to . + /// + public string? BaseDirectory { get; set; } + + /// + /// Directory that contains plugin assemblies. Relative values are resolved against . + /// Defaults to PluginBinaries under the base directory. + /// + public string? PluginsDirectory { get; set; } + + /// + /// Primary prefix used to discover plugin assemblies. If not supplied, the entry assembly name is used. + /// + public string? PrimaryPrefix { get; set; } + + /// + /// Additional prefixes that should be considered when building search patterns. + /// + public IList AdditionalPrefixes => additionalPrefixes; + + /// + /// Explicit plugin ordering expressed as assembly names without extension. + /// Entries that are not discovered will be reported in . + /// + public IList PluginOrder => pluginOrder; + + /// + /// Optional explicit search patterns. When empty, they are derived from prefix settings. + /// + public IList SearchPatterns => searchPatterns; + + /// + /// When true (default) the plugin directory will be created if it does not exist. + /// + public bool EnsureDirectoryExists { get; set; } = true; + + /// + /// Controls whether sub-directories should be scanned. Defaults to true. + /// + public bool RecursiveSearch { get; set; } = true; + + internal string ResolveBaseDirectory() + => string.IsNullOrWhiteSpace(BaseDirectory) + ? AppContext.BaseDirectory + : Path.GetFullPath(BaseDirectory); +} \ No newline at end of file diff --git a/src/StellaOps.Plugin/Hosting/PluginHostResult.cs b/src/StellaOps.Plugin/Hosting/PluginHostResult.cs new file mode 100644 index 00000000..f3b4bec3 --- /dev/null +++ b/src/StellaOps.Plugin/Hosting/PluginHostResult.cs @@ -0,0 +1,26 @@ +using System.Collections.Generic; + +namespace StellaOps.Plugin.Hosting; + +public sealed class PluginHostResult +{ + internal PluginHostResult( + string pluginDirectory, + IReadOnlyList searchPatterns, + IReadOnlyList plugins, + IReadOnlyList missingOrderedPlugins) + { + PluginDirectory = pluginDirectory; + SearchPatterns = searchPatterns; + Plugins = plugins; + MissingOrderedPlugins = missingOrderedPlugins; + } + + public string PluginDirectory { get; } + + public IReadOnlyList SearchPatterns { get; } + + public IReadOnlyList Plugins { get; } + + public IReadOnlyList MissingOrderedPlugins { get; } +} \ No newline at end of file diff --git a/src/StellaOps.Plugin/Hosting/PluginLoadContext.cs b/src/StellaOps.Plugin/Hosting/PluginLoadContext.cs new file mode 100644 index 00000000..ee905403 --- /dev/null +++ b/src/StellaOps.Plugin/Hosting/PluginLoadContext.cs @@ -0,0 +1,79 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Reflection; +using System.Runtime.Loader; + +namespace StellaOps.Plugin.Hosting; + +internal sealed class PluginLoadContext : AssemblyLoadContext +{ + private readonly AssemblyDependencyResolver resolver; + private readonly IEnumerable hostAssemblies; + + public PluginLoadContext(string pluginPath) + : base(isCollectible: false) + { + resolver = new AssemblyDependencyResolver(pluginPath); + hostAssemblies = AssemblyLoadContext.Default.Assemblies; + } + + protected override Assembly? Load(AssemblyName assemblyName) + { + // Attempt to reuse assemblies that already exist in the default context when versions are compatible. + var existing = hostAssemblies.FirstOrDefault(a => string.Equals( + a.GetName().Name, + assemblyName.Name, + StringComparison.OrdinalIgnoreCase)); + + if (existing != null && IsCompatible(existing.GetName(), assemblyName)) + { + return existing; + } + + var assemblyPath = resolver.ResolveAssemblyToPath(assemblyName); + if (!string.IsNullOrEmpty(assemblyPath)) + { + return LoadFromAssemblyPath(assemblyPath); + } + + return null; + } + + protected override IntPtr LoadUnmanagedDll(string unmanagedDllName) + { + var libraryPath = resolver.ResolveUnmanagedDllToPath(unmanagedDllName); + if (!string.IsNullOrEmpty(libraryPath)) + { + return LoadUnmanagedDllFromPath(libraryPath); + } + + return IntPtr.Zero; + } + + private static bool IsCompatible(AssemblyName hostAssembly, AssemblyName pluginAssembly) + { + if (hostAssembly.Version == pluginAssembly.Version) + { + return true; + } + + if (hostAssembly.Version is null || pluginAssembly.Version is null) + { + return false; + } + + if (hostAssembly.Version.Major == pluginAssembly.Version.Major && + hostAssembly.Version.Minor >= pluginAssembly.Version.Minor) + { + return true; + } + + if (hostAssembly.Version.Major >= pluginAssembly.Version.Major) + { + return true; + } + + return false; + } +} \ No newline at end of file diff --git a/src/StellaOps.Plugin/Internal/ReflectionExtensions.cs b/src/StellaOps.Plugin/Internal/ReflectionExtensions.cs new file mode 100644 index 00000000..2d391eeb --- /dev/null +++ b/src/StellaOps.Plugin/Internal/ReflectionExtensions.cs @@ -0,0 +1,21 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Reflection; + +namespace StellaOps.Plugin.Internal; + +internal static class ReflectionExtensions +{ + public static IEnumerable GetLoadableTypes(this Assembly assembly) + { + try + { + return assembly.GetTypes(); + } + catch (ReflectionTypeLoadException ex) + { + return ex.Types.Where(static t => t is not null)!; + } + } +} \ No newline at end of file diff --git a/src/StellaOps.Plugin/PluginContracts.cs b/src/StellaOps.Plugin/PluginContracts.cs new file mode 100644 index 00000000..924ca656 --- /dev/null +++ b/src/StellaOps.Plugin/PluginContracts.cs @@ -0,0 +1,172 @@ +using StellaOps.Plugin.Hosting; +using System; +using System.Collections.Generic; +using System.IO; +using System.Reflection; +using System.Threading; +using System.Linq; +using System.Threading.Tasks; + +namespace StellaOps.Plugin; + +public interface IAvailabilityPlugin +{ + string Name { get; } + bool IsAvailable(IServiceProvider services); +} + +public interface IFeedConnector +{ + string SourceName { get; } + Task FetchAsync(IServiceProvider services, CancellationToken cancellationToken); + Task ParseAsync(IServiceProvider services, CancellationToken cancellationToken); + Task MapAsync(IServiceProvider services, CancellationToken cancellationToken); +} + +public interface IFeedExporter +{ + string Name { get; } + Task ExportAsync(IServiceProvider services, CancellationToken cancellationToken); +} + +public interface IConnectorPlugin : IAvailabilityPlugin +{ + IFeedConnector Create(IServiceProvider services); +} + +public interface IExporterPlugin : IAvailabilityPlugin +{ + IFeedExporter Create(IServiceProvider services); +} + +public sealed class PluginCatalog +{ + private readonly List _assemblies = new(); + private readonly HashSet _assemblyLocations = new(StringComparer.OrdinalIgnoreCase); + + public PluginCatalog AddAssembly(Assembly assembly) + { + if (assembly == null) throw new ArgumentNullException(nameof(assembly)); + if (_assemblies.Contains(assembly)) + { + return this; + } + + _assemblies.Add(assembly); + if (!string.IsNullOrWhiteSpace(assembly.Location)) + { + _assemblyLocations.Add(Path.GetFullPath(assembly.Location)); + } + return this; + } + + public PluginCatalog AddFromDirectory(string directory, string searchPattern = "StellaOps.Feedser.*.dll") + { + if (string.IsNullOrWhiteSpace(directory)) throw new ArgumentException("Directory is required", nameof(directory)); + + var fullDirectory = Path.GetFullPath(directory); + var options = new PluginHostOptions + { + PluginsDirectory = fullDirectory, + EnsureDirectoryExists = false, + RecursiveSearch = false, + }; + options.SearchPatterns.Add(searchPattern); + + var result = PluginHost.LoadPlugins(options); + + foreach (var plugin in result.Plugins) + { + AddAssembly(plugin.Assembly); + } + + return this; + } + + public IReadOnlyList GetConnectorPlugins() => PluginLoader.LoadPlugins(_assemblies); + + public IReadOnlyList GetExporterPlugins() => PluginLoader.LoadPlugins(_assemblies); + + public IReadOnlyList GetAvailableConnectorPlugins(IServiceProvider services) + => FilterAvailable(GetConnectorPlugins(), services); + + public IReadOnlyList GetAvailableExporterPlugins(IServiceProvider services) + => FilterAvailable(GetExporterPlugins(), services); + + private static IReadOnlyList FilterAvailable(IEnumerable plugins, IServiceProvider services) + where TPlugin : IAvailabilityPlugin + { + var list = new List(); + foreach (var plugin in plugins) + { + try + { + if (plugin.IsAvailable(services)) + { + list.Add(plugin); + } + } + catch + { + // Treat exceptions as plugin not available. + } + } + return list; + } +} + +public static class PluginLoader +{ + public static IReadOnlyList LoadPlugins(IEnumerable assemblies) + where TPlugin : class + { + if (assemblies == null) throw new ArgumentNullException(nameof(assemblies)); + + var plugins = new List(); + var seen = new HashSet(StringComparer.OrdinalIgnoreCase); + + foreach (var assembly in assemblies) + { + foreach (var candidate in SafeGetTypes(assembly)) + { + if (candidate.IsAbstract || candidate.IsInterface) + { + continue; + } + + if (!typeof(TPlugin).IsAssignableFrom(candidate)) + { + continue; + } + + if (Activator.CreateInstance(candidate) is not TPlugin plugin) + { + continue; + } + + var key = candidate.FullName ?? candidate.Name; + if (key is null || !seen.Add(key)) + { + continue; + } + + plugins.Add(plugin); + } + } + + return plugins; + } + + private static IEnumerable SafeGetTypes(Assembly assembly) + { + try + { + return assembly.GetTypes(); + } + catch (ReflectionTypeLoadException ex) + { + return ex.Types.Where(t => t is not null)!; + } + } +} + diff --git a/src/StellaOps.Plugin/StellaOps.Plugin.csproj b/src/StellaOps.Plugin/StellaOps.Plugin.csproj new file mode 100644 index 00000000..cee2eaba --- /dev/null +++ b/src/StellaOps.Plugin/StellaOps.Plugin.csproj @@ -0,0 +1,19 @@ + + + + net10.0 + enable + enable + + + + + + + + + + + + + \ No newline at end of file diff --git a/src/StellaOps.Web/AGENTS.md b/src/StellaOps.Web/AGENTS.md new file mode 100644 index 00000000..03209b69 --- /dev/null +++ b/src/StellaOps.Web/AGENTS.md @@ -0,0 +1,24 @@ +# StellaOps Web Frontend + +## Mission +Design and build the StellaOps web user experience that surfaces backend capabilities (Authority, Feedser, Exporters) through an offline-friendly Angular application. + +## Team Composition +- **UX Specialist** – defines user journeys, interaction patterns, accessibility guidelines, and visual design language. +- **Angular Engineers** – implement the SPA, integrate with backend APIs, and ensure deterministic builds suitable for air-gapped deployments. + +## Operating Principles +- Favor modular Angular architecture (feature modules, shared UI kit) with strong typing via latest TypeScript/Angular releases. +- Align UI flows with backend contracts; coordinate with Authority and Feedser teams for API changes. +- Keep assets and build outputs deterministic and cacheable for Offline Kit packaging. +- Track work using the local `TASKS.md` board; keep statuses (TODO/DOING/REVIEW/BLOCKED/DONE) up to date. + +## Key Paths +- `src/StellaOps.Web` — Angular workspace (to be scaffolded). +- `docs/` — UX specs and mockups (to be added). +- `ops/` — Web deployment manifests for air-gapped environments (future). + +## Coordination +- Sync with DevEx for project scaffolding and build pipelines. +- Partner with Docs Guild to translate UX decisions into operator guides. +- Collaborate with Security Guild to validate authentication flows and session handling. diff --git a/src/StellaOps.Web/TASKS.md b/src/StellaOps.Web/TASKS.md new file mode 100644 index 00000000..65306a0d --- /dev/null +++ b/src/StellaOps.Web/TASKS.md @@ -0,0 +1,5 @@ +# StellaOps Web Task Board (UTC 2025-10-10) + +| ID | Status | Owner(s) | Depends on | Description | Exit Criteria | +|----|--------|----------|------------|-------------|---------------| +| WEB1.TRIVY-SETTINGS | TODO | UX Specialist, Angular Eng | Backend `/exporters/trivy-db` contract | Implement Trivy DB exporter settings panel with `publishFull`, `publishDelta`, `includeFull`, `includeDelta` toggles and “Run export now” action using future `/exporters/trivy-db/settings` API. | ✅ Panel wired to mocked API; ✅ Overrides persisted via settings endpoint; ✅ Manual run button reuses overrides. | diff --git a/src/StellaOps.sln b/src/StellaOps.sln new file mode 100644 index 00000000..4c2d8122 --- /dev/null +++ b/src/StellaOps.sln @@ -0,0 +1,1302 @@ + +Microsoft Visual Studio Solution File, Format Version 12.00 +# Visual Studio Version 17 +VisualStudioVersion = 17.0.31903.59 +MinimumVisualStudioVersion = 10.0.40219.1 +Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "src", "src", "{827E0CD3-B72D-47B6-A68D-7590B98EB39B}" +EndProject +Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "StellaOps.Authority", "StellaOps.Authority", "{361838C4-72E2-1C48-5D76-CA6D1A861242}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "StellaOps.Auth.Abstractions", "StellaOps.Authority\StellaOps.Auth.Abstractions\StellaOps.Auth.Abstractions.csproj", "{D9F91EA0-8AF5-452A-86D8-52BACB2E39CB}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "StellaOps.Auth.Client", "StellaOps.Authority\StellaOps.Auth.Client\StellaOps.Auth.Client.csproj", "{5DBE2E9E-9905-47CE-B8DC-B25409AF1EF2}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "StellaOps.Configuration", "StellaOps.Configuration\StellaOps.Configuration.csproj", "{8BCEAAFC-9168-4CC0-AFDB-177E5F7C15C6}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "StellaOps.Auth.ServerIntegration", "StellaOps.Authority\StellaOps.Auth.ServerIntegration\StellaOps.Auth.ServerIntegration.csproj", "{46D35B4F-6A04-47FF-958B-5E6A73FCC059}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "StellaOps.DependencyInjection", "StellaOps.DependencyInjection\StellaOps.DependencyInjection.csproj", "{44A1241B-8ECF-4AFA-9972-452C39AD43D6}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "StellaOps.Authority", "StellaOps.Authority\StellaOps.Authority\StellaOps.Authority.csproj", "{85AB3BB7-C493-4387-B39A-EB299AC37312}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "StellaOps.Authority.Plugins.Abstractions", "StellaOps.Authority\StellaOps.Authority.Plugins.Abstractions\StellaOps.Authority.Plugins.Abstractions.csproj", "{5C5E91CA-3F98-4E9A-922B-F6415EABD1A3}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "StellaOps.Authority.Plugin.Standard", "StellaOps.Authority\StellaOps.Authority.Plugin.Standard\StellaOps.Authority.Plugin.Standard.csproj", "{93DB06DC-B254-48A9-8F2C-6130A5658F27}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "StellaOps.Plugin", "StellaOps.Plugin\StellaOps.Plugin.csproj", "{03CA315C-8AA1-4CEA-A28B-5EB35C586F4A}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "StellaOps.Cli", "StellaOps.Cli\StellaOps.Cli.csproj", "{40094279-250C-42AE-992A-856718FEFBAC}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "StellaOps.Cli.Tests", "StellaOps.Cli.Tests\StellaOps.Cli.Tests.csproj", "{B2967228-F8F7-4931-B257-1C63CB58CE1D}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "StellaOps.Feedser.Testing", "StellaOps.Feedser.Testing\StellaOps.Feedser.Testing.csproj", "{6D52EC2B-0A1A-4693-A8EE-5AB32A4A3ED9}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "StellaOps.Feedser.Source.Common", "StellaOps.Feedser.Source.Common\StellaOps.Feedser.Source.Common.csproj", "{37F203A3-624E-4794-9C99-16CAC22C17DF}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "StellaOps.Feedser.Storage.Mongo", "StellaOps.Feedser.Storage.Mongo\StellaOps.Feedser.Storage.Mongo.csproj", "{3FF93987-A30A-4D50-8815-7CF3BB7CAE05}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "StellaOps.Feedser.Core", "StellaOps.Feedser.Core\StellaOps.Feedser.Core.csproj", "{AACE8717-0760-42F2-A225-8FCCE876FB65}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "StellaOps.Feedser.Models", "StellaOps.Feedser.Models\StellaOps.Feedser.Models.csproj", "{4AAD6965-E879-44AD-A8ED-E1D713A3CD6D}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "StellaOps.Feedser.Normalization", "StellaOps.Feedser.Normalization\StellaOps.Feedser.Normalization.csproj", "{85D82A87-1F4A-4B1B-8422-5B7A7B7704E3}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "StellaOps.Feedser.Core.Tests", "StellaOps.Feedser.Core.Tests\StellaOps.Feedser.Core.Tests.csproj", "{FE227DF2-875D-4BEA-A4E0-14EA7F3EC1D0}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "StellaOps.Feedser.Exporter.Json", "StellaOps.Feedser.Exporter.Json\StellaOps.Feedser.Exporter.Json.csproj", "{D0FB54BA-4D14-4A32-B09F-7EC94F369460}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "StellaOps.Feedser.Exporter.Json.Tests", "StellaOps.Feedser.Exporter.Json.Tests\StellaOps.Feedser.Exporter.Json.Tests.csproj", "{69C9E010-CBDD-4B89-84CF-7AB56D6A078A}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "StellaOps.Feedser.Exporter.TrivyDb", "StellaOps.Feedser.Exporter.TrivyDb\StellaOps.Feedser.Exporter.TrivyDb.csproj", "{E471176A-E1F3-4DE5-8D30-0865903A217A}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "StellaOps.Feedser.Exporter.TrivyDb.Tests", "StellaOps.Feedser.Exporter.TrivyDb.Tests\StellaOps.Feedser.Exporter.TrivyDb.Tests.csproj", "{FA013511-DF20-45F7-8077-EBA2D6224D64}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "StellaOps.Feedser.Merge", "StellaOps.Feedser.Merge\StellaOps.Feedser.Merge.csproj", "{B9F84697-54FE-4648-B173-EE3D904FFA4D}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "StellaOps.Feedser.Merge.Tests", "StellaOps.Feedser.Merge.Tests\StellaOps.Feedser.Merge.Tests.csproj", "{6751A76C-8ED8-40F4-AE2B-069DB31395FE}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "StellaOps.Feedser.Models.Tests", "StellaOps.Feedser.Models.Tests\StellaOps.Feedser.Models.Tests.csproj", "{DDBFA2EF-9CAE-473F-A438-369CAC25C66A}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "StellaOps.Feedser.Normalization.Tests", "StellaOps.Feedser.Normalization.Tests\StellaOps.Feedser.Normalization.Tests.csproj", "{063DE5E1-C8FE-47D0-A12A-22A25CDF2C22}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "StellaOps.Feedser.Source.Acsc", "StellaOps.Feedser.Source.Acsc\StellaOps.Feedser.Source.Acsc.csproj", "{35350FAB-FC51-4FE8-81FB-011003134C37}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "StellaOps.Feedser.Source.Cccs", "StellaOps.Feedser.Source.Cccs\StellaOps.Feedser.Source.Cccs.csproj", "{1BFC95B4-4C8A-44B2-903A-11FBCAAB9519}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "StellaOps.Feedser.Source.CertBund", "StellaOps.Feedser.Source.CertBund\StellaOps.Feedser.Source.CertBund.csproj", "{C4A65377-22F7-4D15-92A3-4F05847D167E}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "StellaOps.Feedser.Source.CertCc", "StellaOps.Feedser.Source.CertCc\StellaOps.Feedser.Source.CertCc.csproj", "{BDDE59E1-C643-4C87-8608-0F9A7A54DE09}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "StellaOps.Feedser.Source.CertFr", "StellaOps.Feedser.Source.CertFr\StellaOps.Feedser.Source.CertFr.csproj", "{0CC116C8-A7E5-4B94-9688-32920177FF97}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "StellaOps.Feedser.Source.CertFr.Tests", "StellaOps.Feedser.Source.CertFr.Tests\StellaOps.Feedser.Source.CertFr.Tests.csproj", "{E8862F6E-85C1-4FDB-AA92-0BB489B7EA1E}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "StellaOps.Feedser.Source.CertIn", "StellaOps.Feedser.Source.CertIn\StellaOps.Feedser.Source.CertIn.csproj", "{84DEDF05-A5BD-4644-86B9-6B7918FE3F31}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "StellaOps.Feedser.Source.CertIn.Tests", "StellaOps.Feedser.Source.CertIn.Tests\StellaOps.Feedser.Source.CertIn.Tests.csproj", "{9DEB1F54-94B5-40C4-AC44-220E680B016D}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "StellaOps.Feedser.Source.Common.Tests", "StellaOps.Feedser.Source.Common.Tests\StellaOps.Feedser.Source.Common.Tests.csproj", "{7C3E87F2-93D8-4968-95E3-52C46947D46C}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "StellaOps.Feedser.Source.Cve", "StellaOps.Feedser.Source.Cve\StellaOps.Feedser.Source.Cve.csproj", "{C0504D97-9BCD-4AE4-B0DC-B31C17B150F2}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "StellaOps.Feedser.Source.Distro.Debian", "StellaOps.Feedser.Source.Distro.Debian\StellaOps.Feedser.Source.Distro.Debian.csproj", "{31B05493-104F-437F-9FA7-CA5286CE697C}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "StellaOps.Feedser.Source.Distro.Debian.Tests", "StellaOps.Feedser.Source.Distro.Debian.Tests\StellaOps.Feedser.Source.Distro.Debian.Tests.csproj", "{937AF12E-D770-4534-8FF8-C59042609C2A}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "StellaOps.Feedser.Source.Distro.RedHat", "StellaOps.Feedser.Source.Distro.RedHat\StellaOps.Feedser.Source.Distro.RedHat.csproj", "{5A028B04-9D76-470B-B5B3-766CE4CE860C}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "StellaOps.Feedser.Source.Distro.RedHat.Tests", "StellaOps.Feedser.Source.Distro.RedHat.Tests\StellaOps.Feedser.Source.Distro.RedHat.Tests.csproj", "{749DE4C8-F733-43F8-B2A8-6649E71C7570}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "StellaOps.Feedser.Source.Distro.Suse", "StellaOps.Feedser.Source.Distro.Suse\StellaOps.Feedser.Source.Distro.Suse.csproj", "{56D2C79E-2737-4FF9-9D19-150065F568D5}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "StellaOps.Feedser.Source.Distro.Suse.Tests", "StellaOps.Feedser.Source.Distro.Suse.Tests\StellaOps.Feedser.Source.Distro.Suse.Tests.csproj", "{E41F6DC4-68B5-4EE3-97AE-801D725A2C13}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "StellaOps.Feedser.Source.Distro.Ubuntu", "StellaOps.Feedser.Source.Distro.Ubuntu\StellaOps.Feedser.Source.Distro.Ubuntu.csproj", "{285F1D0F-501F-4E2E-8FA0-F2CF28AE3798}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "StellaOps.Feedser.Source.Distro.Ubuntu.Tests", "StellaOps.Feedser.Source.Distro.Ubuntu.Tests\StellaOps.Feedser.Source.Distro.Ubuntu.Tests.csproj", "{26055403-C7F5-4709-8813-0F7387102791}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "StellaOps.Feedser.Source.Ghsa", "StellaOps.Feedser.Source.Ghsa\StellaOps.Feedser.Source.Ghsa.csproj", "{0C00D0DA-C4C3-4B23-941F-A3DB2DBF33AF}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "StellaOps.Feedser.Source.Ics.Cisa", "StellaOps.Feedser.Source.Ics.Cisa\StellaOps.Feedser.Source.Ics.Cisa.csproj", "{258327E9-431E-475C-933B-50893676E452}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "StellaOps.Feedser.Source.Ics.Kaspersky", "StellaOps.Feedser.Source.Ics.Kaspersky\StellaOps.Feedser.Source.Ics.Kaspersky.csproj", "{42AF60C8-A5E1-40E0-86F8-98256364AF6F}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "StellaOps.Feedser.Source.Ics.Kaspersky.Tests", "StellaOps.Feedser.Source.Ics.Kaspersky.Tests\StellaOps.Feedser.Source.Ics.Kaspersky.Tests.csproj", "{88C6A9C3-B433-4C36-8767-429C8C2396F8}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "StellaOps.Feedser.Source.Jvn", "StellaOps.Feedser.Source.Jvn\StellaOps.Feedser.Source.Jvn.csproj", "{6B7099AB-01BF-4EC4-87D0-5C9C032266DE}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "StellaOps.Feedser.Source.Jvn.Tests", "StellaOps.Feedser.Source.Jvn.Tests\StellaOps.Feedser.Source.Jvn.Tests.csproj", "{14C918EA-693E-41FE-ACAE-2E82DF077BEA}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "StellaOps.Feedser.Source.Kev", "StellaOps.Feedser.Source.Kev\StellaOps.Feedser.Source.Kev.csproj", "{81111B26-74F6-4912-9084-7115FD119945}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "StellaOps.Feedser.Source.Kisa", "StellaOps.Feedser.Source.Kisa\StellaOps.Feedser.Source.Kisa.csproj", "{80E2D661-FF3E-4A10-A2DF-AFD4F3D433FE}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "StellaOps.Feedser.Source.Nvd", "StellaOps.Feedser.Source.Nvd\StellaOps.Feedser.Source.Nvd.csproj", "{8D0F501D-01B1-4E24-958B-FAF35B267705}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "StellaOps.Feedser.Source.Nvd.Tests", "StellaOps.Feedser.Source.Nvd.Tests\StellaOps.Feedser.Source.Nvd.Tests.csproj", "{5BA91095-7F10-4717-B296-49DFBFC1C9C2}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "StellaOps.Feedser.Source.Osv", "StellaOps.Feedser.Source.Osv\StellaOps.Feedser.Source.Osv.csproj", "{99616566-4EF1-4DC7-B655-825FE43D203D}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "StellaOps.Feedser.Source.Osv.Tests", "StellaOps.Feedser.Source.Osv.Tests\StellaOps.Feedser.Source.Osv.Tests.csproj", "{EE3C03AD-E604-4C57-9B78-CF7F49FBFCB0}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "StellaOps.Feedser.Source.Ru.Bdu", "StellaOps.Feedser.Source.Ru.Bdu\StellaOps.Feedser.Source.Ru.Bdu.csproj", "{A3B19095-2D95-4B09-B07E-2C082C72394B}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "StellaOps.Feedser.Source.Ru.Nkcki", "StellaOps.Feedser.Source.Ru.Nkcki\StellaOps.Feedser.Source.Ru.Nkcki.csproj", "{807837AF-B392-4589-ADF1-3FDB34D6C5BF}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "StellaOps.Feedser.Source.Vndr.Adobe", "StellaOps.Feedser.Source.Vndr.Adobe\StellaOps.Feedser.Source.Vndr.Adobe.csproj", "{64EAFDCF-8283-4D5C-AC78-7969D5FE926A}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "StellaOps.Feedser.Source.Vndr.Adobe.Tests", "StellaOps.Feedser.Source.Vndr.Adobe.Tests\StellaOps.Feedser.Source.Vndr.Adobe.Tests.csproj", "{68F4D8A1-E32F-487A-B460-325F36989BE3}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "StellaOps.Feedser.Source.Vndr.Apple", "StellaOps.Feedser.Source.Vndr.Apple\StellaOps.Feedser.Source.Vndr.Apple.csproj", "{4A3DA4AE-7B88-4674-A7E2-F5D42B8256F2}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "StellaOps.Feedser.Source.Vndr.Chromium", "StellaOps.Feedser.Source.Vndr.Chromium\StellaOps.Feedser.Source.Vndr.Chromium.csproj", "{606C751B-7CF1-47CF-A25C-9248A55C814F}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "StellaOps.Feedser.Source.Vndr.Chromium.Tests", "StellaOps.Feedser.Source.Vndr.Chromium.Tests\StellaOps.Feedser.Source.Vndr.Chromium.Tests.csproj", "{0BE44D0A-CC4B-4E84-8AF3-D8D99551C431}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "StellaOps.Feedser.Source.Vndr.Cisco", "StellaOps.Feedser.Source.Vndr.Cisco\StellaOps.Feedser.Source.Vndr.Cisco.csproj", "{CC4CCE5F-55BC-4745-A204-4FA92BC1BADC}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "StellaOps.Feedser.Source.Vndr.Msrc", "StellaOps.Feedser.Source.Vndr.Msrc\StellaOps.Feedser.Source.Vndr.Msrc.csproj", "{5CCE0DB7-C115-4B21-A7AE-C8488C22A853}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "StellaOps.Feedser.Source.Vndr.Oracle", "StellaOps.Feedser.Source.Vndr.Oracle\StellaOps.Feedser.Source.Vndr.Oracle.csproj", "{A09C9E66-5496-47EC-8B23-9EEB7CBDC75E}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "StellaOps.Feedser.Source.Vndr.Oracle.Tests", "StellaOps.Feedser.Source.Vndr.Oracle.Tests\StellaOps.Feedser.Source.Vndr.Oracle.Tests.csproj", "{06DC817F-A936-4F83-8929-E00622B32245}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "StellaOps.Feedser.Source.Vndr.Vmware", "StellaOps.Feedser.Source.Vndr.Vmware\StellaOps.Feedser.Source.Vndr.Vmware.csproj", "{2C999476-0291-4161-B3E9-1AA99A3B1139}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "StellaOps.Feedser.Source.Vndr.Vmware.Tests", "StellaOps.Feedser.Source.Vndr.Vmware.Tests\StellaOps.Feedser.Source.Vndr.Vmware.Tests.csproj", "{476EAADA-1B39-4049-ABE4-CCAC21FFE9E2}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "StellaOps.Feedser.Storage.Mongo.Tests", "StellaOps.Feedser.Storage.Mongo.Tests\StellaOps.Feedser.Storage.Mongo.Tests.csproj", "{0EF56124-E6E8-4E89-95DD-5A5D5FF05A98}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "StellaOps.Feedser.WebService", "StellaOps.Feedser.WebService\StellaOps.Feedser.WebService.csproj", "{0DBB9FC4-2E46-4C3E-BE88-2A8DCB59DB7D}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "StellaOps.Feedser.WebService.Tests", "StellaOps.Feedser.WebService.Tests\StellaOps.Feedser.WebService.Tests.csproj", "{8A40142F-E8C8-4E86-BE70-7DD4AB1FFDEE}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "StellaOps.Configuration.Tests", "StellaOps.Configuration.Tests\StellaOps.Configuration.Tests.csproj", "{C9D20F74-EE5F-4C9E-9AB1-C03E90B34F92}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "StellaOps.Authority.Plugins.Abstractions.Tests", "StellaOps.Authority\StellaOps.Authority.Plugins.Abstractions.Tests\StellaOps.Authority.Plugins.Abstractions.Tests.csproj", "{50140A32-6D3C-47DB-983A-7166CBA51845}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "StellaOps.Authority.Tests", "StellaOps.Authority\StellaOps.Authority.Tests\StellaOps.Authority.Tests.csproj", "{031979F2-6ABA-444F-A6A4-80115DC487CE}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "StellaOps.Authority.Plugin.Standard.Tests", "StellaOps.Authority\StellaOps.Authority.Plugin.Standard.Tests\StellaOps.Authority.Plugin.Standard.Tests.csproj", "{D71B0DA5-80A3-419E-898D-40E77A9A7F19}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "StellaOps.Authority.Storage.Mongo", "StellaOps.Authority\StellaOps.Authority.Storage.Mongo\StellaOps.Authority.Storage.Mongo.csproj", "{B2C877D9-B521-4901-8817-76B5DAA62FCE}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "StellaOps.Auth.Abstractions.Tests", "StellaOps.Authority\StellaOps.Auth.Abstractions.Tests\StellaOps.Auth.Abstractions.Tests.csproj", "{08D3B6D0-3CE8-4F24-A6F1-BCAB01AD6278}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "StellaOps.Auth.ServerIntegration.Tests", "StellaOps.Authority\StellaOps.Auth.ServerIntegration.Tests\StellaOps.Auth.ServerIntegration.Tests.csproj", "{7116DD6B-2491-49E1-AB27-5210E949F753}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "StellaOps.Auth.Client.Tests", "StellaOps.Authority\StellaOps.Auth.Client.Tests\StellaOps.Auth.Client.Tests.csproj", "{7DBE31A6-D2FD-499E-B675-4092723175AD}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "StellaOps.Feedser.Source.Kev.Tests", "StellaOps.Feedser.Source.Kev.Tests\StellaOps.Feedser.Source.Kev.Tests.csproj", "{D99E6EAE-D278-4480-AA67-85F025383E47}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "StellaOps.Feedser.Source.Cve.Tests", "StellaOps.Feedser.Source.Cve.Tests\StellaOps.Feedser.Source.Cve.Tests.csproj", "{D3825714-3DDA-44B7-A99C-5F3E65716691}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "StellaOps.Feedser.Source.Ghsa.Tests", "StellaOps.Feedser.Source.Ghsa.Tests\StellaOps.Feedser.Source.Ghsa.Tests.csproj", "{FAB78D21-7372-48FE-B2C3-DE1807F1157D}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "StellaOps.Cryptography", "StellaOps.Cryptography\StellaOps.Cryptography.csproj", "{EADFA337-B0FA-4712-A24A-7C08235BDF98}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "StellaOps.Cryptography.Tests", "StellaOps.Cryptography.Tests\StellaOps.Cryptography.Tests.csproj", "{110F7EC2-3149-4D1B-A972-E69E79F1EBF5}" +EndProject +Global + GlobalSection(SolutionConfigurationPlatforms) = preSolution + Debug|Any CPU = Debug|Any CPU + Debug|x64 = Debug|x64 + Debug|x86 = Debug|x86 + Release|Any CPU = Release|Any CPU + Release|x64 = Release|x64 + Release|x86 = Release|x86 + EndGlobalSection + GlobalSection(ProjectConfigurationPlatforms) = postSolution + {D9F91EA0-8AF5-452A-86D8-52BACB2E39CB}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {D9F91EA0-8AF5-452A-86D8-52BACB2E39CB}.Debug|Any CPU.Build.0 = Debug|Any CPU + {D9F91EA0-8AF5-452A-86D8-52BACB2E39CB}.Debug|x64.ActiveCfg = Debug|Any CPU + {D9F91EA0-8AF5-452A-86D8-52BACB2E39CB}.Debug|x64.Build.0 = Debug|Any CPU + {D9F91EA0-8AF5-452A-86D8-52BACB2E39CB}.Debug|x86.ActiveCfg = Debug|Any CPU + {D9F91EA0-8AF5-452A-86D8-52BACB2E39CB}.Debug|x86.Build.0 = Debug|Any CPU + {D9F91EA0-8AF5-452A-86D8-52BACB2E39CB}.Release|Any CPU.ActiveCfg = Release|Any CPU + {D9F91EA0-8AF5-452A-86D8-52BACB2E39CB}.Release|Any CPU.Build.0 = Release|Any CPU + {D9F91EA0-8AF5-452A-86D8-52BACB2E39CB}.Release|x64.ActiveCfg = Release|Any CPU + {D9F91EA0-8AF5-452A-86D8-52BACB2E39CB}.Release|x64.Build.0 = Release|Any CPU + {D9F91EA0-8AF5-452A-86D8-52BACB2E39CB}.Release|x86.ActiveCfg = Release|Any CPU + {D9F91EA0-8AF5-452A-86D8-52BACB2E39CB}.Release|x86.Build.0 = Release|Any CPU + {5DBE2E9E-9905-47CE-B8DC-B25409AF1EF2}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {5DBE2E9E-9905-47CE-B8DC-B25409AF1EF2}.Debug|Any CPU.Build.0 = Debug|Any CPU + {5DBE2E9E-9905-47CE-B8DC-B25409AF1EF2}.Debug|x64.ActiveCfg = Debug|Any CPU + {5DBE2E9E-9905-47CE-B8DC-B25409AF1EF2}.Debug|x64.Build.0 = Debug|Any CPU + {5DBE2E9E-9905-47CE-B8DC-B25409AF1EF2}.Debug|x86.ActiveCfg = Debug|Any CPU + {5DBE2E9E-9905-47CE-B8DC-B25409AF1EF2}.Debug|x86.Build.0 = Debug|Any CPU + {5DBE2E9E-9905-47CE-B8DC-B25409AF1EF2}.Release|Any CPU.ActiveCfg = Release|Any CPU + {5DBE2E9E-9905-47CE-B8DC-B25409AF1EF2}.Release|Any CPU.Build.0 = Release|Any CPU + {5DBE2E9E-9905-47CE-B8DC-B25409AF1EF2}.Release|x64.ActiveCfg = Release|Any CPU + {5DBE2E9E-9905-47CE-B8DC-B25409AF1EF2}.Release|x64.Build.0 = Release|Any CPU + {5DBE2E9E-9905-47CE-B8DC-B25409AF1EF2}.Release|x86.ActiveCfg = Release|Any CPU + {5DBE2E9E-9905-47CE-B8DC-B25409AF1EF2}.Release|x86.Build.0 = Release|Any CPU + {8BCEAAFC-9168-4CC0-AFDB-177E5F7C15C6}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {8BCEAAFC-9168-4CC0-AFDB-177E5F7C15C6}.Debug|Any CPU.Build.0 = Debug|Any CPU + {8BCEAAFC-9168-4CC0-AFDB-177E5F7C15C6}.Debug|x64.ActiveCfg = Debug|Any CPU + {8BCEAAFC-9168-4CC0-AFDB-177E5F7C15C6}.Debug|x64.Build.0 = Debug|Any CPU + {8BCEAAFC-9168-4CC0-AFDB-177E5F7C15C6}.Debug|x86.ActiveCfg = Debug|Any CPU + {8BCEAAFC-9168-4CC0-AFDB-177E5F7C15C6}.Debug|x86.Build.0 = Debug|Any CPU + {8BCEAAFC-9168-4CC0-AFDB-177E5F7C15C6}.Release|Any CPU.ActiveCfg = Release|Any CPU + {8BCEAAFC-9168-4CC0-AFDB-177E5F7C15C6}.Release|Any CPU.Build.0 = Release|Any CPU + {8BCEAAFC-9168-4CC0-AFDB-177E5F7C15C6}.Release|x64.ActiveCfg = Release|Any CPU + {8BCEAAFC-9168-4CC0-AFDB-177E5F7C15C6}.Release|x64.Build.0 = Release|Any CPU + {8BCEAAFC-9168-4CC0-AFDB-177E5F7C15C6}.Release|x86.ActiveCfg = Release|Any CPU + {8BCEAAFC-9168-4CC0-AFDB-177E5F7C15C6}.Release|x86.Build.0 = Release|Any CPU + {46D35B4F-6A04-47FF-958B-5E6A73FCC059}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {46D35B4F-6A04-47FF-958B-5E6A73FCC059}.Debug|Any CPU.Build.0 = Debug|Any CPU + {46D35B4F-6A04-47FF-958B-5E6A73FCC059}.Debug|x64.ActiveCfg = Debug|Any CPU + {46D35B4F-6A04-47FF-958B-5E6A73FCC059}.Debug|x64.Build.0 = Debug|Any CPU + {46D35B4F-6A04-47FF-958B-5E6A73FCC059}.Debug|x86.ActiveCfg = Debug|Any CPU + {46D35B4F-6A04-47FF-958B-5E6A73FCC059}.Debug|x86.Build.0 = Debug|Any CPU + {46D35B4F-6A04-47FF-958B-5E6A73FCC059}.Release|Any CPU.ActiveCfg = Release|Any CPU + {46D35B4F-6A04-47FF-958B-5E6A73FCC059}.Release|Any CPU.Build.0 = Release|Any CPU + {46D35B4F-6A04-47FF-958B-5E6A73FCC059}.Release|x64.ActiveCfg = Release|Any CPU + {46D35B4F-6A04-47FF-958B-5E6A73FCC059}.Release|x64.Build.0 = Release|Any CPU + {46D35B4F-6A04-47FF-958B-5E6A73FCC059}.Release|x86.ActiveCfg = Release|Any CPU + {46D35B4F-6A04-47FF-958B-5E6A73FCC059}.Release|x86.Build.0 = Release|Any CPU + {44A1241B-8ECF-4AFA-9972-452C39AD43D6}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {44A1241B-8ECF-4AFA-9972-452C39AD43D6}.Debug|Any CPU.Build.0 = Debug|Any CPU + {44A1241B-8ECF-4AFA-9972-452C39AD43D6}.Debug|x64.ActiveCfg = Debug|Any CPU + {44A1241B-8ECF-4AFA-9972-452C39AD43D6}.Debug|x64.Build.0 = Debug|Any CPU + {44A1241B-8ECF-4AFA-9972-452C39AD43D6}.Debug|x86.ActiveCfg = Debug|Any CPU + {44A1241B-8ECF-4AFA-9972-452C39AD43D6}.Debug|x86.Build.0 = Debug|Any CPU + {44A1241B-8ECF-4AFA-9972-452C39AD43D6}.Release|Any CPU.ActiveCfg = Release|Any CPU + {44A1241B-8ECF-4AFA-9972-452C39AD43D6}.Release|Any CPU.Build.0 = Release|Any CPU + {44A1241B-8ECF-4AFA-9972-452C39AD43D6}.Release|x64.ActiveCfg = Release|Any CPU + {44A1241B-8ECF-4AFA-9972-452C39AD43D6}.Release|x64.Build.0 = Release|Any CPU + {44A1241B-8ECF-4AFA-9972-452C39AD43D6}.Release|x86.ActiveCfg = Release|Any CPU + {44A1241B-8ECF-4AFA-9972-452C39AD43D6}.Release|x86.Build.0 = Release|Any CPU + {85AB3BB7-C493-4387-B39A-EB299AC37312}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {85AB3BB7-C493-4387-B39A-EB299AC37312}.Debug|Any CPU.Build.0 = Debug|Any CPU + {85AB3BB7-C493-4387-B39A-EB299AC37312}.Debug|x64.ActiveCfg = Debug|Any CPU + {85AB3BB7-C493-4387-B39A-EB299AC37312}.Debug|x64.Build.0 = Debug|Any CPU + {85AB3BB7-C493-4387-B39A-EB299AC37312}.Debug|x86.ActiveCfg = Debug|Any CPU + {85AB3BB7-C493-4387-B39A-EB299AC37312}.Debug|x86.Build.0 = Debug|Any CPU + {85AB3BB7-C493-4387-B39A-EB299AC37312}.Release|Any CPU.ActiveCfg = Release|Any CPU + {85AB3BB7-C493-4387-B39A-EB299AC37312}.Release|Any CPU.Build.0 = Release|Any CPU + {85AB3BB7-C493-4387-B39A-EB299AC37312}.Release|x64.ActiveCfg = Release|Any CPU + {85AB3BB7-C493-4387-B39A-EB299AC37312}.Release|x64.Build.0 = Release|Any CPU + {85AB3BB7-C493-4387-B39A-EB299AC37312}.Release|x86.ActiveCfg = Release|Any CPU + {85AB3BB7-C493-4387-B39A-EB299AC37312}.Release|x86.Build.0 = Release|Any CPU + {5C5E91CA-3F98-4E9A-922B-F6415EABD1A3}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {5C5E91CA-3F98-4E9A-922B-F6415EABD1A3}.Debug|Any CPU.Build.0 = Debug|Any CPU + {5C5E91CA-3F98-4E9A-922B-F6415EABD1A3}.Debug|x64.ActiveCfg = Debug|Any CPU + {5C5E91CA-3F98-4E9A-922B-F6415EABD1A3}.Debug|x64.Build.0 = Debug|Any CPU + {5C5E91CA-3F98-4E9A-922B-F6415EABD1A3}.Debug|x86.ActiveCfg = Debug|Any CPU + {5C5E91CA-3F98-4E9A-922B-F6415EABD1A3}.Debug|x86.Build.0 = Debug|Any CPU + {5C5E91CA-3F98-4E9A-922B-F6415EABD1A3}.Release|Any CPU.ActiveCfg = Release|Any CPU + {5C5E91CA-3F98-4E9A-922B-F6415EABD1A3}.Release|Any CPU.Build.0 = Release|Any CPU + {5C5E91CA-3F98-4E9A-922B-F6415EABD1A3}.Release|x64.ActiveCfg = Release|Any CPU + {5C5E91CA-3F98-4E9A-922B-F6415EABD1A3}.Release|x64.Build.0 = Release|Any CPU + {5C5E91CA-3F98-4E9A-922B-F6415EABD1A3}.Release|x86.ActiveCfg = Release|Any CPU + {5C5E91CA-3F98-4E9A-922B-F6415EABD1A3}.Release|x86.Build.0 = Release|Any CPU + {93DB06DC-B254-48A9-8F2C-6130A5658F27}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {93DB06DC-B254-48A9-8F2C-6130A5658F27}.Debug|Any CPU.Build.0 = Debug|Any CPU + {93DB06DC-B254-48A9-8F2C-6130A5658F27}.Debug|x64.ActiveCfg = Debug|Any CPU + {93DB06DC-B254-48A9-8F2C-6130A5658F27}.Debug|x64.Build.0 = Debug|Any CPU + {93DB06DC-B254-48A9-8F2C-6130A5658F27}.Debug|x86.ActiveCfg = Debug|Any CPU + {93DB06DC-B254-48A9-8F2C-6130A5658F27}.Debug|x86.Build.0 = Debug|Any CPU + {93DB06DC-B254-48A9-8F2C-6130A5658F27}.Release|Any CPU.ActiveCfg = Release|Any CPU + {93DB06DC-B254-48A9-8F2C-6130A5658F27}.Release|Any CPU.Build.0 = Release|Any CPU + {93DB06DC-B254-48A9-8F2C-6130A5658F27}.Release|x64.ActiveCfg = Release|Any CPU + {93DB06DC-B254-48A9-8F2C-6130A5658F27}.Release|x64.Build.0 = Release|Any CPU + {93DB06DC-B254-48A9-8F2C-6130A5658F27}.Release|x86.ActiveCfg = Release|Any CPU + {93DB06DC-B254-48A9-8F2C-6130A5658F27}.Release|x86.Build.0 = Release|Any CPU + {03CA315C-8AA1-4CEA-A28B-5EB35C586F4A}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {03CA315C-8AA1-4CEA-A28B-5EB35C586F4A}.Debug|Any CPU.Build.0 = Debug|Any CPU + {03CA315C-8AA1-4CEA-A28B-5EB35C586F4A}.Debug|x64.ActiveCfg = Debug|Any CPU + {03CA315C-8AA1-4CEA-A28B-5EB35C586F4A}.Debug|x64.Build.0 = Debug|Any CPU + {03CA315C-8AA1-4CEA-A28B-5EB35C586F4A}.Debug|x86.ActiveCfg = Debug|Any CPU + {03CA315C-8AA1-4CEA-A28B-5EB35C586F4A}.Debug|x86.Build.0 = Debug|Any CPU + {03CA315C-8AA1-4CEA-A28B-5EB35C586F4A}.Release|Any CPU.ActiveCfg = Release|Any CPU + {03CA315C-8AA1-4CEA-A28B-5EB35C586F4A}.Release|Any CPU.Build.0 = Release|Any CPU + {03CA315C-8AA1-4CEA-A28B-5EB35C586F4A}.Release|x64.ActiveCfg = Release|Any CPU + {03CA315C-8AA1-4CEA-A28B-5EB35C586F4A}.Release|x64.Build.0 = Release|Any CPU + {03CA315C-8AA1-4CEA-A28B-5EB35C586F4A}.Release|x86.ActiveCfg = Release|Any CPU + {03CA315C-8AA1-4CEA-A28B-5EB35C586F4A}.Release|x86.Build.0 = Release|Any CPU + {40094279-250C-42AE-992A-856718FEFBAC}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {40094279-250C-42AE-992A-856718FEFBAC}.Debug|Any CPU.Build.0 = Debug|Any CPU + {40094279-250C-42AE-992A-856718FEFBAC}.Debug|x64.ActiveCfg = Debug|Any CPU + {40094279-250C-42AE-992A-856718FEFBAC}.Debug|x64.Build.0 = Debug|Any CPU + {40094279-250C-42AE-992A-856718FEFBAC}.Debug|x86.ActiveCfg = Debug|Any CPU + {40094279-250C-42AE-992A-856718FEFBAC}.Debug|x86.Build.0 = Debug|Any CPU + {40094279-250C-42AE-992A-856718FEFBAC}.Release|Any CPU.ActiveCfg = Release|Any CPU + {40094279-250C-42AE-992A-856718FEFBAC}.Release|Any CPU.Build.0 = Release|Any CPU + {40094279-250C-42AE-992A-856718FEFBAC}.Release|x64.ActiveCfg = Release|Any CPU + {40094279-250C-42AE-992A-856718FEFBAC}.Release|x64.Build.0 = Release|Any CPU + {40094279-250C-42AE-992A-856718FEFBAC}.Release|x86.ActiveCfg = Release|Any CPU + {40094279-250C-42AE-992A-856718FEFBAC}.Release|x86.Build.0 = Release|Any CPU + {B2967228-F8F7-4931-B257-1C63CB58CE1D}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {B2967228-F8F7-4931-B257-1C63CB58CE1D}.Debug|Any CPU.Build.0 = Debug|Any CPU + {B2967228-F8F7-4931-B257-1C63CB58CE1D}.Debug|x64.ActiveCfg = Debug|Any CPU + {B2967228-F8F7-4931-B257-1C63CB58CE1D}.Debug|x64.Build.0 = Debug|Any CPU + {B2967228-F8F7-4931-B257-1C63CB58CE1D}.Debug|x86.ActiveCfg = Debug|Any CPU + {B2967228-F8F7-4931-B257-1C63CB58CE1D}.Debug|x86.Build.0 = Debug|Any CPU + {B2967228-F8F7-4931-B257-1C63CB58CE1D}.Release|Any CPU.ActiveCfg = Release|Any CPU + {B2967228-F8F7-4931-B257-1C63CB58CE1D}.Release|Any CPU.Build.0 = Release|Any CPU + {B2967228-F8F7-4931-B257-1C63CB58CE1D}.Release|x64.ActiveCfg = Release|Any CPU + {B2967228-F8F7-4931-B257-1C63CB58CE1D}.Release|x64.Build.0 = Release|Any CPU + {B2967228-F8F7-4931-B257-1C63CB58CE1D}.Release|x86.ActiveCfg = Release|Any CPU + {B2967228-F8F7-4931-B257-1C63CB58CE1D}.Release|x86.Build.0 = Release|Any CPU + {6D52EC2B-0A1A-4693-A8EE-5AB32A4A3ED9}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {6D52EC2B-0A1A-4693-A8EE-5AB32A4A3ED9}.Debug|Any CPU.Build.0 = Debug|Any CPU + {6D52EC2B-0A1A-4693-A8EE-5AB32A4A3ED9}.Debug|x64.ActiveCfg = Debug|Any CPU + {6D52EC2B-0A1A-4693-A8EE-5AB32A4A3ED9}.Debug|x64.Build.0 = Debug|Any CPU + {6D52EC2B-0A1A-4693-A8EE-5AB32A4A3ED9}.Debug|x86.ActiveCfg = Debug|Any CPU + {6D52EC2B-0A1A-4693-A8EE-5AB32A4A3ED9}.Debug|x86.Build.0 = Debug|Any CPU + {6D52EC2B-0A1A-4693-A8EE-5AB32A4A3ED9}.Release|Any CPU.ActiveCfg = Release|Any CPU + {6D52EC2B-0A1A-4693-A8EE-5AB32A4A3ED9}.Release|Any CPU.Build.0 = Release|Any CPU + {6D52EC2B-0A1A-4693-A8EE-5AB32A4A3ED9}.Release|x64.ActiveCfg = Release|Any CPU + {6D52EC2B-0A1A-4693-A8EE-5AB32A4A3ED9}.Release|x64.Build.0 = Release|Any CPU + {6D52EC2B-0A1A-4693-A8EE-5AB32A4A3ED9}.Release|x86.ActiveCfg = Release|Any CPU + {6D52EC2B-0A1A-4693-A8EE-5AB32A4A3ED9}.Release|x86.Build.0 = Release|Any CPU + {37F203A3-624E-4794-9C99-16CAC22C17DF}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {37F203A3-624E-4794-9C99-16CAC22C17DF}.Debug|Any CPU.Build.0 = Debug|Any CPU + {37F203A3-624E-4794-9C99-16CAC22C17DF}.Debug|x64.ActiveCfg = Debug|Any CPU + {37F203A3-624E-4794-9C99-16CAC22C17DF}.Debug|x64.Build.0 = Debug|Any CPU + {37F203A3-624E-4794-9C99-16CAC22C17DF}.Debug|x86.ActiveCfg = Debug|Any CPU + {37F203A3-624E-4794-9C99-16CAC22C17DF}.Debug|x86.Build.0 = Debug|Any CPU + {37F203A3-624E-4794-9C99-16CAC22C17DF}.Release|Any CPU.ActiveCfg = Release|Any CPU + {37F203A3-624E-4794-9C99-16CAC22C17DF}.Release|Any CPU.Build.0 = Release|Any CPU + {37F203A3-624E-4794-9C99-16CAC22C17DF}.Release|x64.ActiveCfg = Release|Any CPU + {37F203A3-624E-4794-9C99-16CAC22C17DF}.Release|x64.Build.0 = Release|Any CPU + {37F203A3-624E-4794-9C99-16CAC22C17DF}.Release|x86.ActiveCfg = Release|Any CPU + {37F203A3-624E-4794-9C99-16CAC22C17DF}.Release|x86.Build.0 = Release|Any CPU + {3FF93987-A30A-4D50-8815-7CF3BB7CAE05}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {3FF93987-A30A-4D50-8815-7CF3BB7CAE05}.Debug|Any CPU.Build.0 = Debug|Any CPU + {3FF93987-A30A-4D50-8815-7CF3BB7CAE05}.Debug|x64.ActiveCfg = Debug|Any CPU + {3FF93987-A30A-4D50-8815-7CF3BB7CAE05}.Debug|x64.Build.0 = Debug|Any CPU + {3FF93987-A30A-4D50-8815-7CF3BB7CAE05}.Debug|x86.ActiveCfg = Debug|Any CPU + {3FF93987-A30A-4D50-8815-7CF3BB7CAE05}.Debug|x86.Build.0 = Debug|Any CPU + {3FF93987-A30A-4D50-8815-7CF3BB7CAE05}.Release|Any CPU.ActiveCfg = Release|Any CPU + {3FF93987-A30A-4D50-8815-7CF3BB7CAE05}.Release|Any CPU.Build.0 = Release|Any CPU + {3FF93987-A30A-4D50-8815-7CF3BB7CAE05}.Release|x64.ActiveCfg = Release|Any CPU + {3FF93987-A30A-4D50-8815-7CF3BB7CAE05}.Release|x64.Build.0 = Release|Any CPU + {3FF93987-A30A-4D50-8815-7CF3BB7CAE05}.Release|x86.ActiveCfg = Release|Any CPU + {3FF93987-A30A-4D50-8815-7CF3BB7CAE05}.Release|x86.Build.0 = Release|Any CPU + {AACE8717-0760-42F2-A225-8FCCE876FB65}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {AACE8717-0760-42F2-A225-8FCCE876FB65}.Debug|Any CPU.Build.0 = Debug|Any CPU + {AACE8717-0760-42F2-A225-8FCCE876FB65}.Debug|x64.ActiveCfg = Debug|Any CPU + {AACE8717-0760-42F2-A225-8FCCE876FB65}.Debug|x64.Build.0 = Debug|Any CPU + {AACE8717-0760-42F2-A225-8FCCE876FB65}.Debug|x86.ActiveCfg = Debug|Any CPU + {AACE8717-0760-42F2-A225-8FCCE876FB65}.Debug|x86.Build.0 = Debug|Any CPU + {AACE8717-0760-42F2-A225-8FCCE876FB65}.Release|Any CPU.ActiveCfg = Release|Any CPU + {AACE8717-0760-42F2-A225-8FCCE876FB65}.Release|Any CPU.Build.0 = Release|Any CPU + {AACE8717-0760-42F2-A225-8FCCE876FB65}.Release|x64.ActiveCfg = Release|Any CPU + {AACE8717-0760-42F2-A225-8FCCE876FB65}.Release|x64.Build.0 = Release|Any CPU + {AACE8717-0760-42F2-A225-8FCCE876FB65}.Release|x86.ActiveCfg = Release|Any CPU + {AACE8717-0760-42F2-A225-8FCCE876FB65}.Release|x86.Build.0 = Release|Any CPU + {4AAD6965-E879-44AD-A8ED-E1D713A3CD6D}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {4AAD6965-E879-44AD-A8ED-E1D713A3CD6D}.Debug|Any CPU.Build.0 = Debug|Any CPU + {4AAD6965-E879-44AD-A8ED-E1D713A3CD6D}.Debug|x64.ActiveCfg = Debug|Any CPU + {4AAD6965-E879-44AD-A8ED-E1D713A3CD6D}.Debug|x64.Build.0 = Debug|Any CPU + {4AAD6965-E879-44AD-A8ED-E1D713A3CD6D}.Debug|x86.ActiveCfg = Debug|Any CPU + {4AAD6965-E879-44AD-A8ED-E1D713A3CD6D}.Debug|x86.Build.0 = Debug|Any CPU + {4AAD6965-E879-44AD-A8ED-E1D713A3CD6D}.Release|Any CPU.ActiveCfg = Release|Any CPU + {4AAD6965-E879-44AD-A8ED-E1D713A3CD6D}.Release|Any CPU.Build.0 = Release|Any CPU + {4AAD6965-E879-44AD-A8ED-E1D713A3CD6D}.Release|x64.ActiveCfg = Release|Any CPU + {4AAD6965-E879-44AD-A8ED-E1D713A3CD6D}.Release|x64.Build.0 = Release|Any CPU + {4AAD6965-E879-44AD-A8ED-E1D713A3CD6D}.Release|x86.ActiveCfg = Release|Any CPU + {4AAD6965-E879-44AD-A8ED-E1D713A3CD6D}.Release|x86.Build.0 = Release|Any CPU + {85D82A87-1F4A-4B1B-8422-5B7A7B7704E3}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {85D82A87-1F4A-4B1B-8422-5B7A7B7704E3}.Debug|Any CPU.Build.0 = Debug|Any CPU + {85D82A87-1F4A-4B1B-8422-5B7A7B7704E3}.Debug|x64.ActiveCfg = Debug|Any CPU + {85D82A87-1F4A-4B1B-8422-5B7A7B7704E3}.Debug|x64.Build.0 = Debug|Any CPU + {85D82A87-1F4A-4B1B-8422-5B7A7B7704E3}.Debug|x86.ActiveCfg = Debug|Any CPU + {85D82A87-1F4A-4B1B-8422-5B7A7B7704E3}.Debug|x86.Build.0 = Debug|Any CPU + {85D82A87-1F4A-4B1B-8422-5B7A7B7704E3}.Release|Any CPU.ActiveCfg = Release|Any CPU + {85D82A87-1F4A-4B1B-8422-5B7A7B7704E3}.Release|Any CPU.Build.0 = Release|Any CPU + {85D82A87-1F4A-4B1B-8422-5B7A7B7704E3}.Release|x64.ActiveCfg = Release|Any CPU + {85D82A87-1F4A-4B1B-8422-5B7A7B7704E3}.Release|x64.Build.0 = Release|Any CPU + {85D82A87-1F4A-4B1B-8422-5B7A7B7704E3}.Release|x86.ActiveCfg = Release|Any CPU + {85D82A87-1F4A-4B1B-8422-5B7A7B7704E3}.Release|x86.Build.0 = Release|Any CPU + {FE227DF2-875D-4BEA-A4E0-14EA7F3EC1D0}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {FE227DF2-875D-4BEA-A4E0-14EA7F3EC1D0}.Debug|Any CPU.Build.0 = Debug|Any CPU + {FE227DF2-875D-4BEA-A4E0-14EA7F3EC1D0}.Debug|x64.ActiveCfg = Debug|Any CPU + {FE227DF2-875D-4BEA-A4E0-14EA7F3EC1D0}.Debug|x64.Build.0 = Debug|Any CPU + {FE227DF2-875D-4BEA-A4E0-14EA7F3EC1D0}.Debug|x86.ActiveCfg = Debug|Any CPU + {FE227DF2-875D-4BEA-A4E0-14EA7F3EC1D0}.Debug|x86.Build.0 = Debug|Any CPU + {FE227DF2-875D-4BEA-A4E0-14EA7F3EC1D0}.Release|Any CPU.ActiveCfg = Release|Any CPU + {FE227DF2-875D-4BEA-A4E0-14EA7F3EC1D0}.Release|Any CPU.Build.0 = Release|Any CPU + {FE227DF2-875D-4BEA-A4E0-14EA7F3EC1D0}.Release|x64.ActiveCfg = Release|Any CPU + {FE227DF2-875D-4BEA-A4E0-14EA7F3EC1D0}.Release|x64.Build.0 = Release|Any CPU + {FE227DF2-875D-4BEA-A4E0-14EA7F3EC1D0}.Release|x86.ActiveCfg = Release|Any CPU + {FE227DF2-875D-4BEA-A4E0-14EA7F3EC1D0}.Release|x86.Build.0 = Release|Any CPU + {D0FB54BA-4D14-4A32-B09F-7EC94F369460}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {D0FB54BA-4D14-4A32-B09F-7EC94F369460}.Debug|Any CPU.Build.0 = Debug|Any CPU + {D0FB54BA-4D14-4A32-B09F-7EC94F369460}.Debug|x64.ActiveCfg = Debug|Any CPU + {D0FB54BA-4D14-4A32-B09F-7EC94F369460}.Debug|x64.Build.0 = Debug|Any CPU + {D0FB54BA-4D14-4A32-B09F-7EC94F369460}.Debug|x86.ActiveCfg = Debug|Any CPU + {D0FB54BA-4D14-4A32-B09F-7EC94F369460}.Debug|x86.Build.0 = Debug|Any CPU + {D0FB54BA-4D14-4A32-B09F-7EC94F369460}.Release|Any CPU.ActiveCfg = Release|Any CPU + {D0FB54BA-4D14-4A32-B09F-7EC94F369460}.Release|Any CPU.Build.0 = Release|Any CPU + {D0FB54BA-4D14-4A32-B09F-7EC94F369460}.Release|x64.ActiveCfg = Release|Any CPU + {D0FB54BA-4D14-4A32-B09F-7EC94F369460}.Release|x64.Build.0 = Release|Any CPU + {D0FB54BA-4D14-4A32-B09F-7EC94F369460}.Release|x86.ActiveCfg = Release|Any CPU + {D0FB54BA-4D14-4A32-B09F-7EC94F369460}.Release|x86.Build.0 = Release|Any CPU + {69C9E010-CBDD-4B89-84CF-7AB56D6A078A}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {69C9E010-CBDD-4B89-84CF-7AB56D6A078A}.Debug|Any CPU.Build.0 = Debug|Any CPU + {69C9E010-CBDD-4B89-84CF-7AB56D6A078A}.Debug|x64.ActiveCfg = Debug|Any CPU + {69C9E010-CBDD-4B89-84CF-7AB56D6A078A}.Debug|x64.Build.0 = Debug|Any CPU + {69C9E010-CBDD-4B89-84CF-7AB56D6A078A}.Debug|x86.ActiveCfg = Debug|Any CPU + {69C9E010-CBDD-4B89-84CF-7AB56D6A078A}.Debug|x86.Build.0 = Debug|Any CPU + {69C9E010-CBDD-4B89-84CF-7AB56D6A078A}.Release|Any CPU.ActiveCfg = Release|Any CPU + {69C9E010-CBDD-4B89-84CF-7AB56D6A078A}.Release|Any CPU.Build.0 = Release|Any CPU + {69C9E010-CBDD-4B89-84CF-7AB56D6A078A}.Release|x64.ActiveCfg = Release|Any CPU + {69C9E010-CBDD-4B89-84CF-7AB56D6A078A}.Release|x64.Build.0 = Release|Any CPU + {69C9E010-CBDD-4B89-84CF-7AB56D6A078A}.Release|x86.ActiveCfg = Release|Any CPU + {69C9E010-CBDD-4B89-84CF-7AB56D6A078A}.Release|x86.Build.0 = Release|Any CPU + {E471176A-E1F3-4DE5-8D30-0865903A217A}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {E471176A-E1F3-4DE5-8D30-0865903A217A}.Debug|Any CPU.Build.0 = Debug|Any CPU + {E471176A-E1F3-4DE5-8D30-0865903A217A}.Debug|x64.ActiveCfg = Debug|Any CPU + {E471176A-E1F3-4DE5-8D30-0865903A217A}.Debug|x64.Build.0 = Debug|Any CPU + {E471176A-E1F3-4DE5-8D30-0865903A217A}.Debug|x86.ActiveCfg = Debug|Any CPU + {E471176A-E1F3-4DE5-8D30-0865903A217A}.Debug|x86.Build.0 = Debug|Any CPU + {E471176A-E1F3-4DE5-8D30-0865903A217A}.Release|Any CPU.ActiveCfg = Release|Any CPU + {E471176A-E1F3-4DE5-8D30-0865903A217A}.Release|Any CPU.Build.0 = Release|Any CPU + {E471176A-E1F3-4DE5-8D30-0865903A217A}.Release|x64.ActiveCfg = Release|Any CPU + {E471176A-E1F3-4DE5-8D30-0865903A217A}.Release|x64.Build.0 = Release|Any CPU + {E471176A-E1F3-4DE5-8D30-0865903A217A}.Release|x86.ActiveCfg = Release|Any CPU + {E471176A-E1F3-4DE5-8D30-0865903A217A}.Release|x86.Build.0 = Release|Any CPU + {FA013511-DF20-45F7-8077-EBA2D6224D64}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {FA013511-DF20-45F7-8077-EBA2D6224D64}.Debug|Any CPU.Build.0 = Debug|Any CPU + {FA013511-DF20-45F7-8077-EBA2D6224D64}.Debug|x64.ActiveCfg = Debug|Any CPU + {FA013511-DF20-45F7-8077-EBA2D6224D64}.Debug|x64.Build.0 = Debug|Any CPU + {FA013511-DF20-45F7-8077-EBA2D6224D64}.Debug|x86.ActiveCfg = Debug|Any CPU + {FA013511-DF20-45F7-8077-EBA2D6224D64}.Debug|x86.Build.0 = Debug|Any CPU + {FA013511-DF20-45F7-8077-EBA2D6224D64}.Release|Any CPU.ActiveCfg = Release|Any CPU + {FA013511-DF20-45F7-8077-EBA2D6224D64}.Release|Any CPU.Build.0 = Release|Any CPU + {FA013511-DF20-45F7-8077-EBA2D6224D64}.Release|x64.ActiveCfg = Release|Any CPU + {FA013511-DF20-45F7-8077-EBA2D6224D64}.Release|x64.Build.0 = Release|Any CPU + {FA013511-DF20-45F7-8077-EBA2D6224D64}.Release|x86.ActiveCfg = Release|Any CPU + {FA013511-DF20-45F7-8077-EBA2D6224D64}.Release|x86.Build.0 = Release|Any CPU + {B9F84697-54FE-4648-B173-EE3D904FFA4D}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {B9F84697-54FE-4648-B173-EE3D904FFA4D}.Debug|Any CPU.Build.0 = Debug|Any CPU + {B9F84697-54FE-4648-B173-EE3D904FFA4D}.Debug|x64.ActiveCfg = Debug|Any CPU + {B9F84697-54FE-4648-B173-EE3D904FFA4D}.Debug|x64.Build.0 = Debug|Any CPU + {B9F84697-54FE-4648-B173-EE3D904FFA4D}.Debug|x86.ActiveCfg = Debug|Any CPU + {B9F84697-54FE-4648-B173-EE3D904FFA4D}.Debug|x86.Build.0 = Debug|Any CPU + {B9F84697-54FE-4648-B173-EE3D904FFA4D}.Release|Any CPU.ActiveCfg = Release|Any CPU + {B9F84697-54FE-4648-B173-EE3D904FFA4D}.Release|Any CPU.Build.0 = Release|Any CPU + {B9F84697-54FE-4648-B173-EE3D904FFA4D}.Release|x64.ActiveCfg = Release|Any CPU + {B9F84697-54FE-4648-B173-EE3D904FFA4D}.Release|x64.Build.0 = Release|Any CPU + {B9F84697-54FE-4648-B173-EE3D904FFA4D}.Release|x86.ActiveCfg = Release|Any CPU + {B9F84697-54FE-4648-B173-EE3D904FFA4D}.Release|x86.Build.0 = Release|Any CPU + {6751A76C-8ED8-40F4-AE2B-069DB31395FE}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {6751A76C-8ED8-40F4-AE2B-069DB31395FE}.Debug|Any CPU.Build.0 = Debug|Any CPU + {6751A76C-8ED8-40F4-AE2B-069DB31395FE}.Debug|x64.ActiveCfg = Debug|Any CPU + {6751A76C-8ED8-40F4-AE2B-069DB31395FE}.Debug|x64.Build.0 = Debug|Any CPU + {6751A76C-8ED8-40F4-AE2B-069DB31395FE}.Debug|x86.ActiveCfg = Debug|Any CPU + {6751A76C-8ED8-40F4-AE2B-069DB31395FE}.Debug|x86.Build.0 = Debug|Any CPU + {6751A76C-8ED8-40F4-AE2B-069DB31395FE}.Release|Any CPU.ActiveCfg = Release|Any CPU + {6751A76C-8ED8-40F4-AE2B-069DB31395FE}.Release|Any CPU.Build.0 = Release|Any CPU + {6751A76C-8ED8-40F4-AE2B-069DB31395FE}.Release|x64.ActiveCfg = Release|Any CPU + {6751A76C-8ED8-40F4-AE2B-069DB31395FE}.Release|x64.Build.0 = Release|Any CPU + {6751A76C-8ED8-40F4-AE2B-069DB31395FE}.Release|x86.ActiveCfg = Release|Any CPU + {6751A76C-8ED8-40F4-AE2B-069DB31395FE}.Release|x86.Build.0 = Release|Any CPU + {DDBFA2EF-9CAE-473F-A438-369CAC25C66A}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {DDBFA2EF-9CAE-473F-A438-369CAC25C66A}.Debug|Any CPU.Build.0 = Debug|Any CPU + {DDBFA2EF-9CAE-473F-A438-369CAC25C66A}.Debug|x64.ActiveCfg = Debug|Any CPU + {DDBFA2EF-9CAE-473F-A438-369CAC25C66A}.Debug|x64.Build.0 = Debug|Any CPU + {DDBFA2EF-9CAE-473F-A438-369CAC25C66A}.Debug|x86.ActiveCfg = Debug|Any CPU + {DDBFA2EF-9CAE-473F-A438-369CAC25C66A}.Debug|x86.Build.0 = Debug|Any CPU + {DDBFA2EF-9CAE-473F-A438-369CAC25C66A}.Release|Any CPU.ActiveCfg = Release|Any CPU + {DDBFA2EF-9CAE-473F-A438-369CAC25C66A}.Release|Any CPU.Build.0 = Release|Any CPU + {DDBFA2EF-9CAE-473F-A438-369CAC25C66A}.Release|x64.ActiveCfg = Release|Any CPU + {DDBFA2EF-9CAE-473F-A438-369CAC25C66A}.Release|x64.Build.0 = Release|Any CPU + {DDBFA2EF-9CAE-473F-A438-369CAC25C66A}.Release|x86.ActiveCfg = Release|Any CPU + {DDBFA2EF-9CAE-473F-A438-369CAC25C66A}.Release|x86.Build.0 = Release|Any CPU + {063DE5E1-C8FE-47D0-A12A-22A25CDF2C22}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {063DE5E1-C8FE-47D0-A12A-22A25CDF2C22}.Debug|Any CPU.Build.0 = Debug|Any CPU + {063DE5E1-C8FE-47D0-A12A-22A25CDF2C22}.Debug|x64.ActiveCfg = Debug|Any CPU + {063DE5E1-C8FE-47D0-A12A-22A25CDF2C22}.Debug|x64.Build.0 = Debug|Any CPU + {063DE5E1-C8FE-47D0-A12A-22A25CDF2C22}.Debug|x86.ActiveCfg = Debug|Any CPU + {063DE5E1-C8FE-47D0-A12A-22A25CDF2C22}.Debug|x86.Build.0 = Debug|Any CPU + {063DE5E1-C8FE-47D0-A12A-22A25CDF2C22}.Release|Any CPU.ActiveCfg = Release|Any CPU + {063DE5E1-C8FE-47D0-A12A-22A25CDF2C22}.Release|Any CPU.Build.0 = Release|Any CPU + {063DE5E1-C8FE-47D0-A12A-22A25CDF2C22}.Release|x64.ActiveCfg = Release|Any CPU + {063DE5E1-C8FE-47D0-A12A-22A25CDF2C22}.Release|x64.Build.0 = Release|Any CPU + {063DE5E1-C8FE-47D0-A12A-22A25CDF2C22}.Release|x86.ActiveCfg = Release|Any CPU + {063DE5E1-C8FE-47D0-A12A-22A25CDF2C22}.Release|x86.Build.0 = Release|Any CPU + {35350FAB-FC51-4FE8-81FB-011003134C37}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {35350FAB-FC51-4FE8-81FB-011003134C37}.Debug|Any CPU.Build.0 = Debug|Any CPU + {35350FAB-FC51-4FE8-81FB-011003134C37}.Debug|x64.ActiveCfg = Debug|Any CPU + {35350FAB-FC51-4FE8-81FB-011003134C37}.Debug|x64.Build.0 = Debug|Any CPU + {35350FAB-FC51-4FE8-81FB-011003134C37}.Debug|x86.ActiveCfg = Debug|Any CPU + {35350FAB-FC51-4FE8-81FB-011003134C37}.Debug|x86.Build.0 = Debug|Any CPU + {35350FAB-FC51-4FE8-81FB-011003134C37}.Release|Any CPU.ActiveCfg = Release|Any CPU + {35350FAB-FC51-4FE8-81FB-011003134C37}.Release|Any CPU.Build.0 = Release|Any CPU + {35350FAB-FC51-4FE8-81FB-011003134C37}.Release|x64.ActiveCfg = Release|Any CPU + {35350FAB-FC51-4FE8-81FB-011003134C37}.Release|x64.Build.0 = Release|Any CPU + {35350FAB-FC51-4FE8-81FB-011003134C37}.Release|x86.ActiveCfg = Release|Any CPU + {35350FAB-FC51-4FE8-81FB-011003134C37}.Release|x86.Build.0 = Release|Any CPU + {1BFC95B4-4C8A-44B2-903A-11FBCAAB9519}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {1BFC95B4-4C8A-44B2-903A-11FBCAAB9519}.Debug|Any CPU.Build.0 = Debug|Any CPU + {1BFC95B4-4C8A-44B2-903A-11FBCAAB9519}.Debug|x64.ActiveCfg = Debug|Any CPU + {1BFC95B4-4C8A-44B2-903A-11FBCAAB9519}.Debug|x64.Build.0 = Debug|Any CPU + {1BFC95B4-4C8A-44B2-903A-11FBCAAB9519}.Debug|x86.ActiveCfg = Debug|Any CPU + {1BFC95B4-4C8A-44B2-903A-11FBCAAB9519}.Debug|x86.Build.0 = Debug|Any CPU + {1BFC95B4-4C8A-44B2-903A-11FBCAAB9519}.Release|Any CPU.ActiveCfg = Release|Any CPU + {1BFC95B4-4C8A-44B2-903A-11FBCAAB9519}.Release|Any CPU.Build.0 = Release|Any CPU + {1BFC95B4-4C8A-44B2-903A-11FBCAAB9519}.Release|x64.ActiveCfg = Release|Any CPU + {1BFC95B4-4C8A-44B2-903A-11FBCAAB9519}.Release|x64.Build.0 = Release|Any CPU + {1BFC95B4-4C8A-44B2-903A-11FBCAAB9519}.Release|x86.ActiveCfg = Release|Any CPU + {1BFC95B4-4C8A-44B2-903A-11FBCAAB9519}.Release|x86.Build.0 = Release|Any CPU + {C4A65377-22F7-4D15-92A3-4F05847D167E}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {C4A65377-22F7-4D15-92A3-4F05847D167E}.Debug|Any CPU.Build.0 = Debug|Any CPU + {C4A65377-22F7-4D15-92A3-4F05847D167E}.Debug|x64.ActiveCfg = Debug|Any CPU + {C4A65377-22F7-4D15-92A3-4F05847D167E}.Debug|x64.Build.0 = Debug|Any CPU + {C4A65377-22F7-4D15-92A3-4F05847D167E}.Debug|x86.ActiveCfg = Debug|Any CPU + {C4A65377-22F7-4D15-92A3-4F05847D167E}.Debug|x86.Build.0 = Debug|Any CPU + {C4A65377-22F7-4D15-92A3-4F05847D167E}.Release|Any CPU.ActiveCfg = Release|Any CPU + {C4A65377-22F7-4D15-92A3-4F05847D167E}.Release|Any CPU.Build.0 = Release|Any CPU + {C4A65377-22F7-4D15-92A3-4F05847D167E}.Release|x64.ActiveCfg = Release|Any CPU + {C4A65377-22F7-4D15-92A3-4F05847D167E}.Release|x64.Build.0 = Release|Any CPU + {C4A65377-22F7-4D15-92A3-4F05847D167E}.Release|x86.ActiveCfg = Release|Any CPU + {C4A65377-22F7-4D15-92A3-4F05847D167E}.Release|x86.Build.0 = Release|Any CPU + {BDDE59E1-C643-4C87-8608-0F9A7A54DE09}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {BDDE59E1-C643-4C87-8608-0F9A7A54DE09}.Debug|Any CPU.Build.0 = Debug|Any CPU + {BDDE59E1-C643-4C87-8608-0F9A7A54DE09}.Debug|x64.ActiveCfg = Debug|Any CPU + {BDDE59E1-C643-4C87-8608-0F9A7A54DE09}.Debug|x64.Build.0 = Debug|Any CPU + {BDDE59E1-C643-4C87-8608-0F9A7A54DE09}.Debug|x86.ActiveCfg = Debug|Any CPU + {BDDE59E1-C643-4C87-8608-0F9A7A54DE09}.Debug|x86.Build.0 = Debug|Any CPU + {BDDE59E1-C643-4C87-8608-0F9A7A54DE09}.Release|Any CPU.ActiveCfg = Release|Any CPU + {BDDE59E1-C643-4C87-8608-0F9A7A54DE09}.Release|Any CPU.Build.0 = Release|Any CPU + {BDDE59E1-C643-4C87-8608-0F9A7A54DE09}.Release|x64.ActiveCfg = Release|Any CPU + {BDDE59E1-C643-4C87-8608-0F9A7A54DE09}.Release|x64.Build.0 = Release|Any CPU + {BDDE59E1-C643-4C87-8608-0F9A7A54DE09}.Release|x86.ActiveCfg = Release|Any CPU + {BDDE59E1-C643-4C87-8608-0F9A7A54DE09}.Release|x86.Build.0 = Release|Any CPU + {0CC116C8-A7E5-4B94-9688-32920177FF97}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {0CC116C8-A7E5-4B94-9688-32920177FF97}.Debug|Any CPU.Build.0 = Debug|Any CPU + {0CC116C8-A7E5-4B94-9688-32920177FF97}.Debug|x64.ActiveCfg = Debug|Any CPU + {0CC116C8-A7E5-4B94-9688-32920177FF97}.Debug|x64.Build.0 = Debug|Any CPU + {0CC116C8-A7E5-4B94-9688-32920177FF97}.Debug|x86.ActiveCfg = Debug|Any CPU + {0CC116C8-A7E5-4B94-9688-32920177FF97}.Debug|x86.Build.0 = Debug|Any CPU + {0CC116C8-A7E5-4B94-9688-32920177FF97}.Release|Any CPU.ActiveCfg = Release|Any CPU + {0CC116C8-A7E5-4B94-9688-32920177FF97}.Release|Any CPU.Build.0 = Release|Any CPU + {0CC116C8-A7E5-4B94-9688-32920177FF97}.Release|x64.ActiveCfg = Release|Any CPU + {0CC116C8-A7E5-4B94-9688-32920177FF97}.Release|x64.Build.0 = Release|Any CPU + {0CC116C8-A7E5-4B94-9688-32920177FF97}.Release|x86.ActiveCfg = Release|Any CPU + {0CC116C8-A7E5-4B94-9688-32920177FF97}.Release|x86.Build.0 = Release|Any CPU + {E8862F6E-85C1-4FDB-AA92-0BB489B7EA1E}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {E8862F6E-85C1-4FDB-AA92-0BB489B7EA1E}.Debug|Any CPU.Build.0 = Debug|Any CPU + {E8862F6E-85C1-4FDB-AA92-0BB489B7EA1E}.Debug|x64.ActiveCfg = Debug|Any CPU + {E8862F6E-85C1-4FDB-AA92-0BB489B7EA1E}.Debug|x64.Build.0 = Debug|Any CPU + {E8862F6E-85C1-4FDB-AA92-0BB489B7EA1E}.Debug|x86.ActiveCfg = Debug|Any CPU + {E8862F6E-85C1-4FDB-AA92-0BB489B7EA1E}.Debug|x86.Build.0 = Debug|Any CPU + {E8862F6E-85C1-4FDB-AA92-0BB489B7EA1E}.Release|Any CPU.ActiveCfg = Release|Any CPU + {E8862F6E-85C1-4FDB-AA92-0BB489B7EA1E}.Release|Any CPU.Build.0 = Release|Any CPU + {E8862F6E-85C1-4FDB-AA92-0BB489B7EA1E}.Release|x64.ActiveCfg = Release|Any CPU + {E8862F6E-85C1-4FDB-AA92-0BB489B7EA1E}.Release|x64.Build.0 = Release|Any CPU + {E8862F6E-85C1-4FDB-AA92-0BB489B7EA1E}.Release|x86.ActiveCfg = Release|Any CPU + {E8862F6E-85C1-4FDB-AA92-0BB489B7EA1E}.Release|x86.Build.0 = Release|Any CPU + {84DEDF05-A5BD-4644-86B9-6B7918FE3F31}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {84DEDF05-A5BD-4644-86B9-6B7918FE3F31}.Debug|Any CPU.Build.0 = Debug|Any CPU + {84DEDF05-A5BD-4644-86B9-6B7918FE3F31}.Debug|x64.ActiveCfg = Debug|Any CPU + {84DEDF05-A5BD-4644-86B9-6B7918FE3F31}.Debug|x64.Build.0 = Debug|Any CPU + {84DEDF05-A5BD-4644-86B9-6B7918FE3F31}.Debug|x86.ActiveCfg = Debug|Any CPU + {84DEDF05-A5BD-4644-86B9-6B7918FE3F31}.Debug|x86.Build.0 = Debug|Any CPU + {84DEDF05-A5BD-4644-86B9-6B7918FE3F31}.Release|Any CPU.ActiveCfg = Release|Any CPU + {84DEDF05-A5BD-4644-86B9-6B7918FE3F31}.Release|Any CPU.Build.0 = Release|Any CPU + {84DEDF05-A5BD-4644-86B9-6B7918FE3F31}.Release|x64.ActiveCfg = Release|Any CPU + {84DEDF05-A5BD-4644-86B9-6B7918FE3F31}.Release|x64.Build.0 = Release|Any CPU + {84DEDF05-A5BD-4644-86B9-6B7918FE3F31}.Release|x86.ActiveCfg = Release|Any CPU + {84DEDF05-A5BD-4644-86B9-6B7918FE3F31}.Release|x86.Build.0 = Release|Any CPU + {9DEB1F54-94B5-40C4-AC44-220E680B016D}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {9DEB1F54-94B5-40C4-AC44-220E680B016D}.Debug|Any CPU.Build.0 = Debug|Any CPU + {9DEB1F54-94B5-40C4-AC44-220E680B016D}.Debug|x64.ActiveCfg = Debug|Any CPU + {9DEB1F54-94B5-40C4-AC44-220E680B016D}.Debug|x64.Build.0 = Debug|Any CPU + {9DEB1F54-94B5-40C4-AC44-220E680B016D}.Debug|x86.ActiveCfg = Debug|Any CPU + {9DEB1F54-94B5-40C4-AC44-220E680B016D}.Debug|x86.Build.0 = Debug|Any CPU + {9DEB1F54-94B5-40C4-AC44-220E680B016D}.Release|Any CPU.ActiveCfg = Release|Any CPU + {9DEB1F54-94B5-40C4-AC44-220E680B016D}.Release|Any CPU.Build.0 = Release|Any CPU + {9DEB1F54-94B5-40C4-AC44-220E680B016D}.Release|x64.ActiveCfg = Release|Any CPU + {9DEB1F54-94B5-40C4-AC44-220E680B016D}.Release|x64.Build.0 = Release|Any CPU + {9DEB1F54-94B5-40C4-AC44-220E680B016D}.Release|x86.ActiveCfg = Release|Any CPU + {9DEB1F54-94B5-40C4-AC44-220E680B016D}.Release|x86.Build.0 = Release|Any CPU + {7C3E87F2-93D8-4968-95E3-52C46947D46C}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {7C3E87F2-93D8-4968-95E3-52C46947D46C}.Debug|Any CPU.Build.0 = Debug|Any CPU + {7C3E87F2-93D8-4968-95E3-52C46947D46C}.Debug|x64.ActiveCfg = Debug|Any CPU + {7C3E87F2-93D8-4968-95E3-52C46947D46C}.Debug|x64.Build.0 = Debug|Any CPU + {7C3E87F2-93D8-4968-95E3-52C46947D46C}.Debug|x86.ActiveCfg = Debug|Any CPU + {7C3E87F2-93D8-4968-95E3-52C46947D46C}.Debug|x86.Build.0 = Debug|Any CPU + {7C3E87F2-93D8-4968-95E3-52C46947D46C}.Release|Any CPU.ActiveCfg = Release|Any CPU + {7C3E87F2-93D8-4968-95E3-52C46947D46C}.Release|Any CPU.Build.0 = Release|Any CPU + {7C3E87F2-93D8-4968-95E3-52C46947D46C}.Release|x64.ActiveCfg = Release|Any CPU + {7C3E87F2-93D8-4968-95E3-52C46947D46C}.Release|x64.Build.0 = Release|Any CPU + {7C3E87F2-93D8-4968-95E3-52C46947D46C}.Release|x86.ActiveCfg = Release|Any CPU + {7C3E87F2-93D8-4968-95E3-52C46947D46C}.Release|x86.Build.0 = Release|Any CPU + {C0504D97-9BCD-4AE4-B0DC-B31C17B150F2}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {C0504D97-9BCD-4AE4-B0DC-B31C17B150F2}.Debug|Any CPU.Build.0 = Debug|Any CPU + {C0504D97-9BCD-4AE4-B0DC-B31C17B150F2}.Debug|x64.ActiveCfg = Debug|Any CPU + {C0504D97-9BCD-4AE4-B0DC-B31C17B150F2}.Debug|x64.Build.0 = Debug|Any CPU + {C0504D97-9BCD-4AE4-B0DC-B31C17B150F2}.Debug|x86.ActiveCfg = Debug|Any CPU + {C0504D97-9BCD-4AE4-B0DC-B31C17B150F2}.Debug|x86.Build.0 = Debug|Any CPU + {C0504D97-9BCD-4AE4-B0DC-B31C17B150F2}.Release|Any CPU.ActiveCfg = Release|Any CPU + {C0504D97-9BCD-4AE4-B0DC-B31C17B150F2}.Release|Any CPU.Build.0 = Release|Any CPU + {C0504D97-9BCD-4AE4-B0DC-B31C17B150F2}.Release|x64.ActiveCfg = Release|Any CPU + {C0504D97-9BCD-4AE4-B0DC-B31C17B150F2}.Release|x64.Build.0 = Release|Any CPU + {C0504D97-9BCD-4AE4-B0DC-B31C17B150F2}.Release|x86.ActiveCfg = Release|Any CPU + {C0504D97-9BCD-4AE4-B0DC-B31C17B150F2}.Release|x86.Build.0 = Release|Any CPU + {31B05493-104F-437F-9FA7-CA5286CE697C}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {31B05493-104F-437F-9FA7-CA5286CE697C}.Debug|Any CPU.Build.0 = Debug|Any CPU + {31B05493-104F-437F-9FA7-CA5286CE697C}.Debug|x64.ActiveCfg = Debug|Any CPU + {31B05493-104F-437F-9FA7-CA5286CE697C}.Debug|x64.Build.0 = Debug|Any CPU + {31B05493-104F-437F-9FA7-CA5286CE697C}.Debug|x86.ActiveCfg = Debug|Any CPU + {31B05493-104F-437F-9FA7-CA5286CE697C}.Debug|x86.Build.0 = Debug|Any CPU + {31B05493-104F-437F-9FA7-CA5286CE697C}.Release|Any CPU.ActiveCfg = Release|Any CPU + {31B05493-104F-437F-9FA7-CA5286CE697C}.Release|Any CPU.Build.0 = Release|Any CPU + {31B05493-104F-437F-9FA7-CA5286CE697C}.Release|x64.ActiveCfg = Release|Any CPU + {31B05493-104F-437F-9FA7-CA5286CE697C}.Release|x64.Build.0 = Release|Any CPU + {31B05493-104F-437F-9FA7-CA5286CE697C}.Release|x86.ActiveCfg = Release|Any CPU + {31B05493-104F-437F-9FA7-CA5286CE697C}.Release|x86.Build.0 = Release|Any CPU + {937AF12E-D770-4534-8FF8-C59042609C2A}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {937AF12E-D770-4534-8FF8-C59042609C2A}.Debug|Any CPU.Build.0 = Debug|Any CPU + {937AF12E-D770-4534-8FF8-C59042609C2A}.Debug|x64.ActiveCfg = Debug|Any CPU + {937AF12E-D770-4534-8FF8-C59042609C2A}.Debug|x64.Build.0 = Debug|Any CPU + {937AF12E-D770-4534-8FF8-C59042609C2A}.Debug|x86.ActiveCfg = Debug|Any CPU + {937AF12E-D770-4534-8FF8-C59042609C2A}.Debug|x86.Build.0 = Debug|Any CPU + {937AF12E-D770-4534-8FF8-C59042609C2A}.Release|Any CPU.ActiveCfg = Release|Any CPU + {937AF12E-D770-4534-8FF8-C59042609C2A}.Release|Any CPU.Build.0 = Release|Any CPU + {937AF12E-D770-4534-8FF8-C59042609C2A}.Release|x64.ActiveCfg = Release|Any CPU + {937AF12E-D770-4534-8FF8-C59042609C2A}.Release|x64.Build.0 = Release|Any CPU + {937AF12E-D770-4534-8FF8-C59042609C2A}.Release|x86.ActiveCfg = Release|Any CPU + {937AF12E-D770-4534-8FF8-C59042609C2A}.Release|x86.Build.0 = Release|Any CPU + {5A028B04-9D76-470B-B5B3-766CE4CE860C}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {5A028B04-9D76-470B-B5B3-766CE4CE860C}.Debug|Any CPU.Build.0 = Debug|Any CPU + {5A028B04-9D76-470B-B5B3-766CE4CE860C}.Debug|x64.ActiveCfg = Debug|Any CPU + {5A028B04-9D76-470B-B5B3-766CE4CE860C}.Debug|x64.Build.0 = Debug|Any CPU + {5A028B04-9D76-470B-B5B3-766CE4CE860C}.Debug|x86.ActiveCfg = Debug|Any CPU + {5A028B04-9D76-470B-B5B3-766CE4CE860C}.Debug|x86.Build.0 = Debug|Any CPU + {5A028B04-9D76-470B-B5B3-766CE4CE860C}.Release|Any CPU.ActiveCfg = Release|Any CPU + {5A028B04-9D76-470B-B5B3-766CE4CE860C}.Release|Any CPU.Build.0 = Release|Any CPU + {5A028B04-9D76-470B-B5B3-766CE4CE860C}.Release|x64.ActiveCfg = Release|Any CPU + {5A028B04-9D76-470B-B5B3-766CE4CE860C}.Release|x64.Build.0 = Release|Any CPU + {5A028B04-9D76-470B-B5B3-766CE4CE860C}.Release|x86.ActiveCfg = Release|Any CPU + {5A028B04-9D76-470B-B5B3-766CE4CE860C}.Release|x86.Build.0 = Release|Any CPU + {749DE4C8-F733-43F8-B2A8-6649E71C7570}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {749DE4C8-F733-43F8-B2A8-6649E71C7570}.Debug|Any CPU.Build.0 = Debug|Any CPU + {749DE4C8-F733-43F8-B2A8-6649E71C7570}.Debug|x64.ActiveCfg = Debug|Any CPU + {749DE4C8-F733-43F8-B2A8-6649E71C7570}.Debug|x64.Build.0 = Debug|Any CPU + {749DE4C8-F733-43F8-B2A8-6649E71C7570}.Debug|x86.ActiveCfg = Debug|Any CPU + {749DE4C8-F733-43F8-B2A8-6649E71C7570}.Debug|x86.Build.0 = Debug|Any CPU + {749DE4C8-F733-43F8-B2A8-6649E71C7570}.Release|Any CPU.ActiveCfg = Release|Any CPU + {749DE4C8-F733-43F8-B2A8-6649E71C7570}.Release|Any CPU.Build.0 = Release|Any CPU + {749DE4C8-F733-43F8-B2A8-6649E71C7570}.Release|x64.ActiveCfg = Release|Any CPU + {749DE4C8-F733-43F8-B2A8-6649E71C7570}.Release|x64.Build.0 = Release|Any CPU + {749DE4C8-F733-43F8-B2A8-6649E71C7570}.Release|x86.ActiveCfg = Release|Any CPU + {749DE4C8-F733-43F8-B2A8-6649E71C7570}.Release|x86.Build.0 = Release|Any CPU + {56D2C79E-2737-4FF9-9D19-150065F568D5}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {56D2C79E-2737-4FF9-9D19-150065F568D5}.Debug|Any CPU.Build.0 = Debug|Any CPU + {56D2C79E-2737-4FF9-9D19-150065F568D5}.Debug|x64.ActiveCfg = Debug|Any CPU + {56D2C79E-2737-4FF9-9D19-150065F568D5}.Debug|x64.Build.0 = Debug|Any CPU + {56D2C79E-2737-4FF9-9D19-150065F568D5}.Debug|x86.ActiveCfg = Debug|Any CPU + {56D2C79E-2737-4FF9-9D19-150065F568D5}.Debug|x86.Build.0 = Debug|Any CPU + {56D2C79E-2737-4FF9-9D19-150065F568D5}.Release|Any CPU.ActiveCfg = Release|Any CPU + {56D2C79E-2737-4FF9-9D19-150065F568D5}.Release|Any CPU.Build.0 = Release|Any CPU + {56D2C79E-2737-4FF9-9D19-150065F568D5}.Release|x64.ActiveCfg = Release|Any CPU + {56D2C79E-2737-4FF9-9D19-150065F568D5}.Release|x64.Build.0 = Release|Any CPU + {56D2C79E-2737-4FF9-9D19-150065F568D5}.Release|x86.ActiveCfg = Release|Any CPU + {56D2C79E-2737-4FF9-9D19-150065F568D5}.Release|x86.Build.0 = Release|Any CPU + {E41F6DC4-68B5-4EE3-97AE-801D725A2C13}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {E41F6DC4-68B5-4EE3-97AE-801D725A2C13}.Debug|Any CPU.Build.0 = Debug|Any CPU + {E41F6DC4-68B5-4EE3-97AE-801D725A2C13}.Debug|x64.ActiveCfg = Debug|Any CPU + {E41F6DC4-68B5-4EE3-97AE-801D725A2C13}.Debug|x64.Build.0 = Debug|Any CPU + {E41F6DC4-68B5-4EE3-97AE-801D725A2C13}.Debug|x86.ActiveCfg = Debug|Any CPU + {E41F6DC4-68B5-4EE3-97AE-801D725A2C13}.Debug|x86.Build.0 = Debug|Any CPU + {E41F6DC4-68B5-4EE3-97AE-801D725A2C13}.Release|Any CPU.ActiveCfg = Release|Any CPU + {E41F6DC4-68B5-4EE3-97AE-801D725A2C13}.Release|Any CPU.Build.0 = Release|Any CPU + {E41F6DC4-68B5-4EE3-97AE-801D725A2C13}.Release|x64.ActiveCfg = Release|Any CPU + {E41F6DC4-68B5-4EE3-97AE-801D725A2C13}.Release|x64.Build.0 = Release|Any CPU + {E41F6DC4-68B5-4EE3-97AE-801D725A2C13}.Release|x86.ActiveCfg = Release|Any CPU + {E41F6DC4-68B5-4EE3-97AE-801D725A2C13}.Release|x86.Build.0 = Release|Any CPU + {285F1D0F-501F-4E2E-8FA0-F2CF28AE3798}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {285F1D0F-501F-4E2E-8FA0-F2CF28AE3798}.Debug|Any CPU.Build.0 = Debug|Any CPU + {285F1D0F-501F-4E2E-8FA0-F2CF28AE3798}.Debug|x64.ActiveCfg = Debug|Any CPU + {285F1D0F-501F-4E2E-8FA0-F2CF28AE3798}.Debug|x64.Build.0 = Debug|Any CPU + {285F1D0F-501F-4E2E-8FA0-F2CF28AE3798}.Debug|x86.ActiveCfg = Debug|Any CPU + {285F1D0F-501F-4E2E-8FA0-F2CF28AE3798}.Debug|x86.Build.0 = Debug|Any CPU + {285F1D0F-501F-4E2E-8FA0-F2CF28AE3798}.Release|Any CPU.ActiveCfg = Release|Any CPU + {285F1D0F-501F-4E2E-8FA0-F2CF28AE3798}.Release|Any CPU.Build.0 = Release|Any CPU + {285F1D0F-501F-4E2E-8FA0-F2CF28AE3798}.Release|x64.ActiveCfg = Release|Any CPU + {285F1D0F-501F-4E2E-8FA0-F2CF28AE3798}.Release|x64.Build.0 = Release|Any CPU + {285F1D0F-501F-4E2E-8FA0-F2CF28AE3798}.Release|x86.ActiveCfg = Release|Any CPU + {285F1D0F-501F-4E2E-8FA0-F2CF28AE3798}.Release|x86.Build.0 = Release|Any CPU + {26055403-C7F5-4709-8813-0F7387102791}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {26055403-C7F5-4709-8813-0F7387102791}.Debug|Any CPU.Build.0 = Debug|Any CPU + {26055403-C7F5-4709-8813-0F7387102791}.Debug|x64.ActiveCfg = Debug|Any CPU + {26055403-C7F5-4709-8813-0F7387102791}.Debug|x64.Build.0 = Debug|Any CPU + {26055403-C7F5-4709-8813-0F7387102791}.Debug|x86.ActiveCfg = Debug|Any CPU + {26055403-C7F5-4709-8813-0F7387102791}.Debug|x86.Build.0 = Debug|Any CPU + {26055403-C7F5-4709-8813-0F7387102791}.Release|Any CPU.ActiveCfg = Release|Any CPU + {26055403-C7F5-4709-8813-0F7387102791}.Release|Any CPU.Build.0 = Release|Any CPU + {26055403-C7F5-4709-8813-0F7387102791}.Release|x64.ActiveCfg = Release|Any CPU + {26055403-C7F5-4709-8813-0F7387102791}.Release|x64.Build.0 = Release|Any CPU + {26055403-C7F5-4709-8813-0F7387102791}.Release|x86.ActiveCfg = Release|Any CPU + {26055403-C7F5-4709-8813-0F7387102791}.Release|x86.Build.0 = Release|Any CPU + {0C00D0DA-C4C3-4B23-941F-A3DB2DBF33AF}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {0C00D0DA-C4C3-4B23-941F-A3DB2DBF33AF}.Debug|Any CPU.Build.0 = Debug|Any CPU + {0C00D0DA-C4C3-4B23-941F-A3DB2DBF33AF}.Debug|x64.ActiveCfg = Debug|Any CPU + {0C00D0DA-C4C3-4B23-941F-A3DB2DBF33AF}.Debug|x64.Build.0 = Debug|Any CPU + {0C00D0DA-C4C3-4B23-941F-A3DB2DBF33AF}.Debug|x86.ActiveCfg = Debug|Any CPU + {0C00D0DA-C4C3-4B23-941F-A3DB2DBF33AF}.Debug|x86.Build.0 = Debug|Any CPU + {0C00D0DA-C4C3-4B23-941F-A3DB2DBF33AF}.Release|Any CPU.ActiveCfg = Release|Any CPU + {0C00D0DA-C4C3-4B23-941F-A3DB2DBF33AF}.Release|Any CPU.Build.0 = Release|Any CPU + {0C00D0DA-C4C3-4B23-941F-A3DB2DBF33AF}.Release|x64.ActiveCfg = Release|Any CPU + {0C00D0DA-C4C3-4B23-941F-A3DB2DBF33AF}.Release|x64.Build.0 = Release|Any CPU + {0C00D0DA-C4C3-4B23-941F-A3DB2DBF33AF}.Release|x86.ActiveCfg = Release|Any CPU + {0C00D0DA-C4C3-4B23-941F-A3DB2DBF33AF}.Release|x86.Build.0 = Release|Any CPU + {258327E9-431E-475C-933B-50893676E452}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {258327E9-431E-475C-933B-50893676E452}.Debug|Any CPU.Build.0 = Debug|Any CPU + {258327E9-431E-475C-933B-50893676E452}.Debug|x64.ActiveCfg = Debug|Any CPU + {258327E9-431E-475C-933B-50893676E452}.Debug|x64.Build.0 = Debug|Any CPU + {258327E9-431E-475C-933B-50893676E452}.Debug|x86.ActiveCfg = Debug|Any CPU + {258327E9-431E-475C-933B-50893676E452}.Debug|x86.Build.0 = Debug|Any CPU + {258327E9-431E-475C-933B-50893676E452}.Release|Any CPU.ActiveCfg = Release|Any CPU + {258327E9-431E-475C-933B-50893676E452}.Release|Any CPU.Build.0 = Release|Any CPU + {258327E9-431E-475C-933B-50893676E452}.Release|x64.ActiveCfg = Release|Any CPU + {258327E9-431E-475C-933B-50893676E452}.Release|x64.Build.0 = Release|Any CPU + {258327E9-431E-475C-933B-50893676E452}.Release|x86.ActiveCfg = Release|Any CPU + {258327E9-431E-475C-933B-50893676E452}.Release|x86.Build.0 = Release|Any CPU + {42AF60C8-A5E1-40E0-86F8-98256364AF6F}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {42AF60C8-A5E1-40E0-86F8-98256364AF6F}.Debug|Any CPU.Build.0 = Debug|Any CPU + {42AF60C8-A5E1-40E0-86F8-98256364AF6F}.Debug|x64.ActiveCfg = Debug|Any CPU + {42AF60C8-A5E1-40E0-86F8-98256364AF6F}.Debug|x64.Build.0 = Debug|Any CPU + {42AF60C8-A5E1-40E0-86F8-98256364AF6F}.Debug|x86.ActiveCfg = Debug|Any CPU + {42AF60C8-A5E1-40E0-86F8-98256364AF6F}.Debug|x86.Build.0 = Debug|Any CPU + {42AF60C8-A5E1-40E0-86F8-98256364AF6F}.Release|Any CPU.ActiveCfg = Release|Any CPU + {42AF60C8-A5E1-40E0-86F8-98256364AF6F}.Release|Any CPU.Build.0 = Release|Any CPU + {42AF60C8-A5E1-40E0-86F8-98256364AF6F}.Release|x64.ActiveCfg = Release|Any CPU + {42AF60C8-A5E1-40E0-86F8-98256364AF6F}.Release|x64.Build.0 = Release|Any CPU + {42AF60C8-A5E1-40E0-86F8-98256364AF6F}.Release|x86.ActiveCfg = Release|Any CPU + {42AF60C8-A5E1-40E0-86F8-98256364AF6F}.Release|x86.Build.0 = Release|Any CPU + {88C6A9C3-B433-4C36-8767-429C8C2396F8}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {88C6A9C3-B433-4C36-8767-429C8C2396F8}.Debug|Any CPU.Build.0 = Debug|Any CPU + {88C6A9C3-B433-4C36-8767-429C8C2396F8}.Debug|x64.ActiveCfg = Debug|Any CPU + {88C6A9C3-B433-4C36-8767-429C8C2396F8}.Debug|x64.Build.0 = Debug|Any CPU + {88C6A9C3-B433-4C36-8767-429C8C2396F8}.Debug|x86.ActiveCfg = Debug|Any CPU + {88C6A9C3-B433-4C36-8767-429C8C2396F8}.Debug|x86.Build.0 = Debug|Any CPU + {88C6A9C3-B433-4C36-8767-429C8C2396F8}.Release|Any CPU.ActiveCfg = Release|Any CPU + {88C6A9C3-B433-4C36-8767-429C8C2396F8}.Release|Any CPU.Build.0 = Release|Any CPU + {88C6A9C3-B433-4C36-8767-429C8C2396F8}.Release|x64.ActiveCfg = Release|Any CPU + {88C6A9C3-B433-4C36-8767-429C8C2396F8}.Release|x64.Build.0 = Release|Any CPU + {88C6A9C3-B433-4C36-8767-429C8C2396F8}.Release|x86.ActiveCfg = Release|Any CPU + {88C6A9C3-B433-4C36-8767-429C8C2396F8}.Release|x86.Build.0 = Release|Any CPU + {6B7099AB-01BF-4EC4-87D0-5C9C032266DE}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {6B7099AB-01BF-4EC4-87D0-5C9C032266DE}.Debug|Any CPU.Build.0 = Debug|Any CPU + {6B7099AB-01BF-4EC4-87D0-5C9C032266DE}.Debug|x64.ActiveCfg = Debug|Any CPU + {6B7099AB-01BF-4EC4-87D0-5C9C032266DE}.Debug|x64.Build.0 = Debug|Any CPU + {6B7099AB-01BF-4EC4-87D0-5C9C032266DE}.Debug|x86.ActiveCfg = Debug|Any CPU + {6B7099AB-01BF-4EC4-87D0-5C9C032266DE}.Debug|x86.Build.0 = Debug|Any CPU + {6B7099AB-01BF-4EC4-87D0-5C9C032266DE}.Release|Any CPU.ActiveCfg = Release|Any CPU + {6B7099AB-01BF-4EC4-87D0-5C9C032266DE}.Release|Any CPU.Build.0 = Release|Any CPU + {6B7099AB-01BF-4EC4-87D0-5C9C032266DE}.Release|x64.ActiveCfg = Release|Any CPU + {6B7099AB-01BF-4EC4-87D0-5C9C032266DE}.Release|x64.Build.0 = Release|Any CPU + {6B7099AB-01BF-4EC4-87D0-5C9C032266DE}.Release|x86.ActiveCfg = Release|Any CPU + {6B7099AB-01BF-4EC4-87D0-5C9C032266DE}.Release|x86.Build.0 = Release|Any CPU + {14C918EA-693E-41FE-ACAE-2E82DF077BEA}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {14C918EA-693E-41FE-ACAE-2E82DF077BEA}.Debug|Any CPU.Build.0 = Debug|Any CPU + {14C918EA-693E-41FE-ACAE-2E82DF077BEA}.Debug|x64.ActiveCfg = Debug|Any CPU + {14C918EA-693E-41FE-ACAE-2E82DF077BEA}.Debug|x64.Build.0 = Debug|Any CPU + {14C918EA-693E-41FE-ACAE-2E82DF077BEA}.Debug|x86.ActiveCfg = Debug|Any CPU + {14C918EA-693E-41FE-ACAE-2E82DF077BEA}.Debug|x86.Build.0 = Debug|Any CPU + {14C918EA-693E-41FE-ACAE-2E82DF077BEA}.Release|Any CPU.ActiveCfg = Release|Any CPU + {14C918EA-693E-41FE-ACAE-2E82DF077BEA}.Release|Any CPU.Build.0 = Release|Any CPU + {14C918EA-693E-41FE-ACAE-2E82DF077BEA}.Release|x64.ActiveCfg = Release|Any CPU + {14C918EA-693E-41FE-ACAE-2E82DF077BEA}.Release|x64.Build.0 = Release|Any CPU + {14C918EA-693E-41FE-ACAE-2E82DF077BEA}.Release|x86.ActiveCfg = Release|Any CPU + {14C918EA-693E-41FE-ACAE-2E82DF077BEA}.Release|x86.Build.0 = Release|Any CPU + {81111B26-74F6-4912-9084-7115FD119945}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {81111B26-74F6-4912-9084-7115FD119945}.Debug|Any CPU.Build.0 = Debug|Any CPU + {81111B26-74F6-4912-9084-7115FD119945}.Debug|x64.ActiveCfg = Debug|Any CPU + {81111B26-74F6-4912-9084-7115FD119945}.Debug|x64.Build.0 = Debug|Any CPU + {81111B26-74F6-4912-9084-7115FD119945}.Debug|x86.ActiveCfg = Debug|Any CPU + {81111B26-74F6-4912-9084-7115FD119945}.Debug|x86.Build.0 = Debug|Any CPU + {81111B26-74F6-4912-9084-7115FD119945}.Release|Any CPU.ActiveCfg = Release|Any CPU + {81111B26-74F6-4912-9084-7115FD119945}.Release|Any CPU.Build.0 = Release|Any CPU + {81111B26-74F6-4912-9084-7115FD119945}.Release|x64.ActiveCfg = Release|Any CPU + {81111B26-74F6-4912-9084-7115FD119945}.Release|x64.Build.0 = Release|Any CPU + {81111B26-74F6-4912-9084-7115FD119945}.Release|x86.ActiveCfg = Release|Any CPU + {81111B26-74F6-4912-9084-7115FD119945}.Release|x86.Build.0 = Release|Any CPU + {80E2D661-FF3E-4A10-A2DF-AFD4F3D433FE}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {80E2D661-FF3E-4A10-A2DF-AFD4F3D433FE}.Debug|Any CPU.Build.0 = Debug|Any CPU + {80E2D661-FF3E-4A10-A2DF-AFD4F3D433FE}.Debug|x64.ActiveCfg = Debug|Any CPU + {80E2D661-FF3E-4A10-A2DF-AFD4F3D433FE}.Debug|x64.Build.0 = Debug|Any CPU + {80E2D661-FF3E-4A10-A2DF-AFD4F3D433FE}.Debug|x86.ActiveCfg = Debug|Any CPU + {80E2D661-FF3E-4A10-A2DF-AFD4F3D433FE}.Debug|x86.Build.0 = Debug|Any CPU + {80E2D661-FF3E-4A10-A2DF-AFD4F3D433FE}.Release|Any CPU.ActiveCfg = Release|Any CPU + {80E2D661-FF3E-4A10-A2DF-AFD4F3D433FE}.Release|Any CPU.Build.0 = Release|Any CPU + {80E2D661-FF3E-4A10-A2DF-AFD4F3D433FE}.Release|x64.ActiveCfg = Release|Any CPU + {80E2D661-FF3E-4A10-A2DF-AFD4F3D433FE}.Release|x64.Build.0 = Release|Any CPU + {80E2D661-FF3E-4A10-A2DF-AFD4F3D433FE}.Release|x86.ActiveCfg = Release|Any CPU + {80E2D661-FF3E-4A10-A2DF-AFD4F3D433FE}.Release|x86.Build.0 = Release|Any CPU + {8D0F501D-01B1-4E24-958B-FAF35B267705}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {8D0F501D-01B1-4E24-958B-FAF35B267705}.Debug|Any CPU.Build.0 = Debug|Any CPU + {8D0F501D-01B1-4E24-958B-FAF35B267705}.Debug|x64.ActiveCfg = Debug|Any CPU + {8D0F501D-01B1-4E24-958B-FAF35B267705}.Debug|x64.Build.0 = Debug|Any CPU + {8D0F501D-01B1-4E24-958B-FAF35B267705}.Debug|x86.ActiveCfg = Debug|Any CPU + {8D0F501D-01B1-4E24-958B-FAF35B267705}.Debug|x86.Build.0 = Debug|Any CPU + {8D0F501D-01B1-4E24-958B-FAF35B267705}.Release|Any CPU.ActiveCfg = Release|Any CPU + {8D0F501D-01B1-4E24-958B-FAF35B267705}.Release|Any CPU.Build.0 = Release|Any CPU + {8D0F501D-01B1-4E24-958B-FAF35B267705}.Release|x64.ActiveCfg = Release|Any CPU + {8D0F501D-01B1-4E24-958B-FAF35B267705}.Release|x64.Build.0 = Release|Any CPU + {8D0F501D-01B1-4E24-958B-FAF35B267705}.Release|x86.ActiveCfg = Release|Any CPU + {8D0F501D-01B1-4E24-958B-FAF35B267705}.Release|x86.Build.0 = Release|Any CPU + {5BA91095-7F10-4717-B296-49DFBFC1C9C2}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {5BA91095-7F10-4717-B296-49DFBFC1C9C2}.Debug|Any CPU.Build.0 = Debug|Any CPU + {5BA91095-7F10-4717-B296-49DFBFC1C9C2}.Debug|x64.ActiveCfg = Debug|Any CPU + {5BA91095-7F10-4717-B296-49DFBFC1C9C2}.Debug|x64.Build.0 = Debug|Any CPU + {5BA91095-7F10-4717-B296-49DFBFC1C9C2}.Debug|x86.ActiveCfg = Debug|Any CPU + {5BA91095-7F10-4717-B296-49DFBFC1C9C2}.Debug|x86.Build.0 = Debug|Any CPU + {5BA91095-7F10-4717-B296-49DFBFC1C9C2}.Release|Any CPU.ActiveCfg = Release|Any CPU + {5BA91095-7F10-4717-B296-49DFBFC1C9C2}.Release|Any CPU.Build.0 = Release|Any CPU + {5BA91095-7F10-4717-B296-49DFBFC1C9C2}.Release|x64.ActiveCfg = Release|Any CPU + {5BA91095-7F10-4717-B296-49DFBFC1C9C2}.Release|x64.Build.0 = Release|Any CPU + {5BA91095-7F10-4717-B296-49DFBFC1C9C2}.Release|x86.ActiveCfg = Release|Any CPU + {5BA91095-7F10-4717-B296-49DFBFC1C9C2}.Release|x86.Build.0 = Release|Any CPU + {99616566-4EF1-4DC7-B655-825FE43D203D}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {99616566-4EF1-4DC7-B655-825FE43D203D}.Debug|Any CPU.Build.0 = Debug|Any CPU + {99616566-4EF1-4DC7-B655-825FE43D203D}.Debug|x64.ActiveCfg = Debug|Any CPU + {99616566-4EF1-4DC7-B655-825FE43D203D}.Debug|x64.Build.0 = Debug|Any CPU + {99616566-4EF1-4DC7-B655-825FE43D203D}.Debug|x86.ActiveCfg = Debug|Any CPU + {99616566-4EF1-4DC7-B655-825FE43D203D}.Debug|x86.Build.0 = Debug|Any CPU + {99616566-4EF1-4DC7-B655-825FE43D203D}.Release|Any CPU.ActiveCfg = Release|Any CPU + {99616566-4EF1-4DC7-B655-825FE43D203D}.Release|Any CPU.Build.0 = Release|Any CPU + {99616566-4EF1-4DC7-B655-825FE43D203D}.Release|x64.ActiveCfg = Release|Any CPU + {99616566-4EF1-4DC7-B655-825FE43D203D}.Release|x64.Build.0 = Release|Any CPU + {99616566-4EF1-4DC7-B655-825FE43D203D}.Release|x86.ActiveCfg = Release|Any CPU + {99616566-4EF1-4DC7-B655-825FE43D203D}.Release|x86.Build.0 = Release|Any CPU + {EE3C03AD-E604-4C57-9B78-CF7F49FBFCB0}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {EE3C03AD-E604-4C57-9B78-CF7F49FBFCB0}.Debug|Any CPU.Build.0 = Debug|Any CPU + {EE3C03AD-E604-4C57-9B78-CF7F49FBFCB0}.Debug|x64.ActiveCfg = Debug|Any CPU + {EE3C03AD-E604-4C57-9B78-CF7F49FBFCB0}.Debug|x64.Build.0 = Debug|Any CPU + {EE3C03AD-E604-4C57-9B78-CF7F49FBFCB0}.Debug|x86.ActiveCfg = Debug|Any CPU + {EE3C03AD-E604-4C57-9B78-CF7F49FBFCB0}.Debug|x86.Build.0 = Debug|Any CPU + {EE3C03AD-E604-4C57-9B78-CF7F49FBFCB0}.Release|Any CPU.ActiveCfg = Release|Any CPU + {EE3C03AD-E604-4C57-9B78-CF7F49FBFCB0}.Release|Any CPU.Build.0 = Release|Any CPU + {EE3C03AD-E604-4C57-9B78-CF7F49FBFCB0}.Release|x64.ActiveCfg = Release|Any CPU + {EE3C03AD-E604-4C57-9B78-CF7F49FBFCB0}.Release|x64.Build.0 = Release|Any CPU + {EE3C03AD-E604-4C57-9B78-CF7F49FBFCB0}.Release|x86.ActiveCfg = Release|Any CPU + {EE3C03AD-E604-4C57-9B78-CF7F49FBFCB0}.Release|x86.Build.0 = Release|Any CPU + {A3B19095-2D95-4B09-B07E-2C082C72394B}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {A3B19095-2D95-4B09-B07E-2C082C72394B}.Debug|Any CPU.Build.0 = Debug|Any CPU + {A3B19095-2D95-4B09-B07E-2C082C72394B}.Debug|x64.ActiveCfg = Debug|Any CPU + {A3B19095-2D95-4B09-B07E-2C082C72394B}.Debug|x64.Build.0 = Debug|Any CPU + {A3B19095-2D95-4B09-B07E-2C082C72394B}.Debug|x86.ActiveCfg = Debug|Any CPU + {A3B19095-2D95-4B09-B07E-2C082C72394B}.Debug|x86.Build.0 = Debug|Any CPU + {A3B19095-2D95-4B09-B07E-2C082C72394B}.Release|Any CPU.ActiveCfg = Release|Any CPU + {A3B19095-2D95-4B09-B07E-2C082C72394B}.Release|Any CPU.Build.0 = Release|Any CPU + {A3B19095-2D95-4B09-B07E-2C082C72394B}.Release|x64.ActiveCfg = Release|Any CPU + {A3B19095-2D95-4B09-B07E-2C082C72394B}.Release|x64.Build.0 = Release|Any CPU + {A3B19095-2D95-4B09-B07E-2C082C72394B}.Release|x86.ActiveCfg = Release|Any CPU + {A3B19095-2D95-4B09-B07E-2C082C72394B}.Release|x86.Build.0 = Release|Any CPU + {807837AF-B392-4589-ADF1-3FDB34D6C5BF}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {807837AF-B392-4589-ADF1-3FDB34D6C5BF}.Debug|Any CPU.Build.0 = Debug|Any CPU + {807837AF-B392-4589-ADF1-3FDB34D6C5BF}.Debug|x64.ActiveCfg = Debug|Any CPU + {807837AF-B392-4589-ADF1-3FDB34D6C5BF}.Debug|x64.Build.0 = Debug|Any CPU + {807837AF-B392-4589-ADF1-3FDB34D6C5BF}.Debug|x86.ActiveCfg = Debug|Any CPU + {807837AF-B392-4589-ADF1-3FDB34D6C5BF}.Debug|x86.Build.0 = Debug|Any CPU + {807837AF-B392-4589-ADF1-3FDB34D6C5BF}.Release|Any CPU.ActiveCfg = Release|Any CPU + {807837AF-B392-4589-ADF1-3FDB34D6C5BF}.Release|Any CPU.Build.0 = Release|Any CPU + {807837AF-B392-4589-ADF1-3FDB34D6C5BF}.Release|x64.ActiveCfg = Release|Any CPU + {807837AF-B392-4589-ADF1-3FDB34D6C5BF}.Release|x64.Build.0 = Release|Any CPU + {807837AF-B392-4589-ADF1-3FDB34D6C5BF}.Release|x86.ActiveCfg = Release|Any CPU + {807837AF-B392-4589-ADF1-3FDB34D6C5BF}.Release|x86.Build.0 = Release|Any CPU + {64EAFDCF-8283-4D5C-AC78-7969D5FE926A}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {64EAFDCF-8283-4D5C-AC78-7969D5FE926A}.Debug|Any CPU.Build.0 = Debug|Any CPU + {64EAFDCF-8283-4D5C-AC78-7969D5FE926A}.Debug|x64.ActiveCfg = Debug|Any CPU + {64EAFDCF-8283-4D5C-AC78-7969D5FE926A}.Debug|x64.Build.0 = Debug|Any CPU + {64EAFDCF-8283-4D5C-AC78-7969D5FE926A}.Debug|x86.ActiveCfg = Debug|Any CPU + {64EAFDCF-8283-4D5C-AC78-7969D5FE926A}.Debug|x86.Build.0 = Debug|Any CPU + {64EAFDCF-8283-4D5C-AC78-7969D5FE926A}.Release|Any CPU.ActiveCfg = Release|Any CPU + {64EAFDCF-8283-4D5C-AC78-7969D5FE926A}.Release|Any CPU.Build.0 = Release|Any CPU + {64EAFDCF-8283-4D5C-AC78-7969D5FE926A}.Release|x64.ActiveCfg = Release|Any CPU + {64EAFDCF-8283-4D5C-AC78-7969D5FE926A}.Release|x64.Build.0 = Release|Any CPU + {64EAFDCF-8283-4D5C-AC78-7969D5FE926A}.Release|x86.ActiveCfg = Release|Any CPU + {64EAFDCF-8283-4D5C-AC78-7969D5FE926A}.Release|x86.Build.0 = Release|Any CPU + {68F4D8A1-E32F-487A-B460-325F36989BE3}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {68F4D8A1-E32F-487A-B460-325F36989BE3}.Debug|Any CPU.Build.0 = Debug|Any CPU + {68F4D8A1-E32F-487A-B460-325F36989BE3}.Debug|x64.ActiveCfg = Debug|Any CPU + {68F4D8A1-E32F-487A-B460-325F36989BE3}.Debug|x64.Build.0 = Debug|Any CPU + {68F4D8A1-E32F-487A-B460-325F36989BE3}.Debug|x86.ActiveCfg = Debug|Any CPU + {68F4D8A1-E32F-487A-B460-325F36989BE3}.Debug|x86.Build.0 = Debug|Any CPU + {68F4D8A1-E32F-487A-B460-325F36989BE3}.Release|Any CPU.ActiveCfg = Release|Any CPU + {68F4D8A1-E32F-487A-B460-325F36989BE3}.Release|Any CPU.Build.0 = Release|Any CPU + {68F4D8A1-E32F-487A-B460-325F36989BE3}.Release|x64.ActiveCfg = Release|Any CPU + {68F4D8A1-E32F-487A-B460-325F36989BE3}.Release|x64.Build.0 = Release|Any CPU + {68F4D8A1-E32F-487A-B460-325F36989BE3}.Release|x86.ActiveCfg = Release|Any CPU + {68F4D8A1-E32F-487A-B460-325F36989BE3}.Release|x86.Build.0 = Release|Any CPU + {4A3DA4AE-7B88-4674-A7E2-F5D42B8256F2}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {4A3DA4AE-7B88-4674-A7E2-F5D42B8256F2}.Debug|Any CPU.Build.0 = Debug|Any CPU + {4A3DA4AE-7B88-4674-A7E2-F5D42B8256F2}.Debug|x64.ActiveCfg = Debug|Any CPU + {4A3DA4AE-7B88-4674-A7E2-F5D42B8256F2}.Debug|x64.Build.0 = Debug|Any CPU + {4A3DA4AE-7B88-4674-A7E2-F5D42B8256F2}.Debug|x86.ActiveCfg = Debug|Any CPU + {4A3DA4AE-7B88-4674-A7E2-F5D42B8256F2}.Debug|x86.Build.0 = Debug|Any CPU + {4A3DA4AE-7B88-4674-A7E2-F5D42B8256F2}.Release|Any CPU.ActiveCfg = Release|Any CPU + {4A3DA4AE-7B88-4674-A7E2-F5D42B8256F2}.Release|Any CPU.Build.0 = Release|Any CPU + {4A3DA4AE-7B88-4674-A7E2-F5D42B8256F2}.Release|x64.ActiveCfg = Release|Any CPU + {4A3DA4AE-7B88-4674-A7E2-F5D42B8256F2}.Release|x64.Build.0 = Release|Any CPU + {4A3DA4AE-7B88-4674-A7E2-F5D42B8256F2}.Release|x86.ActiveCfg = Release|Any CPU + {4A3DA4AE-7B88-4674-A7E2-F5D42B8256F2}.Release|x86.Build.0 = Release|Any CPU + {606C751B-7CF1-47CF-A25C-9248A55C814F}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {606C751B-7CF1-47CF-A25C-9248A55C814F}.Debug|Any CPU.Build.0 = Debug|Any CPU + {606C751B-7CF1-47CF-A25C-9248A55C814F}.Debug|x64.ActiveCfg = Debug|Any CPU + {606C751B-7CF1-47CF-A25C-9248A55C814F}.Debug|x64.Build.0 = Debug|Any CPU + {606C751B-7CF1-47CF-A25C-9248A55C814F}.Debug|x86.ActiveCfg = Debug|Any CPU + {606C751B-7CF1-47CF-A25C-9248A55C814F}.Debug|x86.Build.0 = Debug|Any CPU + {606C751B-7CF1-47CF-A25C-9248A55C814F}.Release|Any CPU.ActiveCfg = Release|Any CPU + {606C751B-7CF1-47CF-A25C-9248A55C814F}.Release|Any CPU.Build.0 = Release|Any CPU + {606C751B-7CF1-47CF-A25C-9248A55C814F}.Release|x64.ActiveCfg = Release|Any CPU + {606C751B-7CF1-47CF-A25C-9248A55C814F}.Release|x64.Build.0 = Release|Any CPU + {606C751B-7CF1-47CF-A25C-9248A55C814F}.Release|x86.ActiveCfg = Release|Any CPU + {606C751B-7CF1-47CF-A25C-9248A55C814F}.Release|x86.Build.0 = Release|Any CPU + {0BE44D0A-CC4B-4E84-8AF3-D8D99551C431}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {0BE44D0A-CC4B-4E84-8AF3-D8D99551C431}.Debug|Any CPU.Build.0 = Debug|Any CPU + {0BE44D0A-CC4B-4E84-8AF3-D8D99551C431}.Debug|x64.ActiveCfg = Debug|Any CPU + {0BE44D0A-CC4B-4E84-8AF3-D8D99551C431}.Debug|x64.Build.0 = Debug|Any CPU + {0BE44D0A-CC4B-4E84-8AF3-D8D99551C431}.Debug|x86.ActiveCfg = Debug|Any CPU + {0BE44D0A-CC4B-4E84-8AF3-D8D99551C431}.Debug|x86.Build.0 = Debug|Any CPU + {0BE44D0A-CC4B-4E84-8AF3-D8D99551C431}.Release|Any CPU.ActiveCfg = Release|Any CPU + {0BE44D0A-CC4B-4E84-8AF3-D8D99551C431}.Release|Any CPU.Build.0 = Release|Any CPU + {0BE44D0A-CC4B-4E84-8AF3-D8D99551C431}.Release|x64.ActiveCfg = Release|Any CPU + {0BE44D0A-CC4B-4E84-8AF3-D8D99551C431}.Release|x64.Build.0 = Release|Any CPU + {0BE44D0A-CC4B-4E84-8AF3-D8D99551C431}.Release|x86.ActiveCfg = Release|Any CPU + {0BE44D0A-CC4B-4E84-8AF3-D8D99551C431}.Release|x86.Build.0 = Release|Any CPU + {CC4CCE5F-55BC-4745-A204-4FA92BC1BADC}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {CC4CCE5F-55BC-4745-A204-4FA92BC1BADC}.Debug|Any CPU.Build.0 = Debug|Any CPU + {CC4CCE5F-55BC-4745-A204-4FA92BC1BADC}.Debug|x64.ActiveCfg = Debug|Any CPU + {CC4CCE5F-55BC-4745-A204-4FA92BC1BADC}.Debug|x64.Build.0 = Debug|Any CPU + {CC4CCE5F-55BC-4745-A204-4FA92BC1BADC}.Debug|x86.ActiveCfg = Debug|Any CPU + {CC4CCE5F-55BC-4745-A204-4FA92BC1BADC}.Debug|x86.Build.0 = Debug|Any CPU + {CC4CCE5F-55BC-4745-A204-4FA92BC1BADC}.Release|Any CPU.ActiveCfg = Release|Any CPU + {CC4CCE5F-55BC-4745-A204-4FA92BC1BADC}.Release|Any CPU.Build.0 = Release|Any CPU + {CC4CCE5F-55BC-4745-A204-4FA92BC1BADC}.Release|x64.ActiveCfg = Release|Any CPU + {CC4CCE5F-55BC-4745-A204-4FA92BC1BADC}.Release|x64.Build.0 = Release|Any CPU + {CC4CCE5F-55BC-4745-A204-4FA92BC1BADC}.Release|x86.ActiveCfg = Release|Any CPU + {CC4CCE5F-55BC-4745-A204-4FA92BC1BADC}.Release|x86.Build.0 = Release|Any CPU + {5CCE0DB7-C115-4B21-A7AE-C8488C22A853}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {5CCE0DB7-C115-4B21-A7AE-C8488C22A853}.Debug|Any CPU.Build.0 = Debug|Any CPU + {5CCE0DB7-C115-4B21-A7AE-C8488C22A853}.Debug|x64.ActiveCfg = Debug|Any CPU + {5CCE0DB7-C115-4B21-A7AE-C8488C22A853}.Debug|x64.Build.0 = Debug|Any CPU + {5CCE0DB7-C115-4B21-A7AE-C8488C22A853}.Debug|x86.ActiveCfg = Debug|Any CPU + {5CCE0DB7-C115-4B21-A7AE-C8488C22A853}.Debug|x86.Build.0 = Debug|Any CPU + {5CCE0DB7-C115-4B21-A7AE-C8488C22A853}.Release|Any CPU.ActiveCfg = Release|Any CPU + {5CCE0DB7-C115-4B21-A7AE-C8488C22A853}.Release|Any CPU.Build.0 = Release|Any CPU + {5CCE0DB7-C115-4B21-A7AE-C8488C22A853}.Release|x64.ActiveCfg = Release|Any CPU + {5CCE0DB7-C115-4B21-A7AE-C8488C22A853}.Release|x64.Build.0 = Release|Any CPU + {5CCE0DB7-C115-4B21-A7AE-C8488C22A853}.Release|x86.ActiveCfg = Release|Any CPU + {5CCE0DB7-C115-4B21-A7AE-C8488C22A853}.Release|x86.Build.0 = Release|Any CPU + {A09C9E66-5496-47EC-8B23-9EEB7CBDC75E}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {A09C9E66-5496-47EC-8B23-9EEB7CBDC75E}.Debug|Any CPU.Build.0 = Debug|Any CPU + {A09C9E66-5496-47EC-8B23-9EEB7CBDC75E}.Debug|x64.ActiveCfg = Debug|Any CPU + {A09C9E66-5496-47EC-8B23-9EEB7CBDC75E}.Debug|x64.Build.0 = Debug|Any CPU + {A09C9E66-5496-47EC-8B23-9EEB7CBDC75E}.Debug|x86.ActiveCfg = Debug|Any CPU + {A09C9E66-5496-47EC-8B23-9EEB7CBDC75E}.Debug|x86.Build.0 = Debug|Any CPU + {A09C9E66-5496-47EC-8B23-9EEB7CBDC75E}.Release|Any CPU.ActiveCfg = Release|Any CPU + {A09C9E66-5496-47EC-8B23-9EEB7CBDC75E}.Release|Any CPU.Build.0 = Release|Any CPU + {A09C9E66-5496-47EC-8B23-9EEB7CBDC75E}.Release|x64.ActiveCfg = Release|Any CPU + {A09C9E66-5496-47EC-8B23-9EEB7CBDC75E}.Release|x64.Build.0 = Release|Any CPU + {A09C9E66-5496-47EC-8B23-9EEB7CBDC75E}.Release|x86.ActiveCfg = Release|Any CPU + {A09C9E66-5496-47EC-8B23-9EEB7CBDC75E}.Release|x86.Build.0 = Release|Any CPU + {06DC817F-A936-4F83-8929-E00622B32245}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {06DC817F-A936-4F83-8929-E00622B32245}.Debug|Any CPU.Build.0 = Debug|Any CPU + {06DC817F-A936-4F83-8929-E00622B32245}.Debug|x64.ActiveCfg = Debug|Any CPU + {06DC817F-A936-4F83-8929-E00622B32245}.Debug|x64.Build.0 = Debug|Any CPU + {06DC817F-A936-4F83-8929-E00622B32245}.Debug|x86.ActiveCfg = Debug|Any CPU + {06DC817F-A936-4F83-8929-E00622B32245}.Debug|x86.Build.0 = Debug|Any CPU + {06DC817F-A936-4F83-8929-E00622B32245}.Release|Any CPU.ActiveCfg = Release|Any CPU + {06DC817F-A936-4F83-8929-E00622B32245}.Release|Any CPU.Build.0 = Release|Any CPU + {06DC817F-A936-4F83-8929-E00622B32245}.Release|x64.ActiveCfg = Release|Any CPU + {06DC817F-A936-4F83-8929-E00622B32245}.Release|x64.Build.0 = Release|Any CPU + {06DC817F-A936-4F83-8929-E00622B32245}.Release|x86.ActiveCfg = Release|Any CPU + {06DC817F-A936-4F83-8929-E00622B32245}.Release|x86.Build.0 = Release|Any CPU + {2C999476-0291-4161-B3E9-1AA99A3B1139}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {2C999476-0291-4161-B3E9-1AA99A3B1139}.Debug|Any CPU.Build.0 = Debug|Any CPU + {2C999476-0291-4161-B3E9-1AA99A3B1139}.Debug|x64.ActiveCfg = Debug|Any CPU + {2C999476-0291-4161-B3E9-1AA99A3B1139}.Debug|x64.Build.0 = Debug|Any CPU + {2C999476-0291-4161-B3E9-1AA99A3B1139}.Debug|x86.ActiveCfg = Debug|Any CPU + {2C999476-0291-4161-B3E9-1AA99A3B1139}.Debug|x86.Build.0 = Debug|Any CPU + {2C999476-0291-4161-B3E9-1AA99A3B1139}.Release|Any CPU.ActiveCfg = Release|Any CPU + {2C999476-0291-4161-B3E9-1AA99A3B1139}.Release|Any CPU.Build.0 = Release|Any CPU + {2C999476-0291-4161-B3E9-1AA99A3B1139}.Release|x64.ActiveCfg = Release|Any CPU + {2C999476-0291-4161-B3E9-1AA99A3B1139}.Release|x64.Build.0 = Release|Any CPU + {2C999476-0291-4161-B3E9-1AA99A3B1139}.Release|x86.ActiveCfg = Release|Any CPU + {2C999476-0291-4161-B3E9-1AA99A3B1139}.Release|x86.Build.0 = Release|Any CPU + {476EAADA-1B39-4049-ABE4-CCAC21FFE9E2}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {476EAADA-1B39-4049-ABE4-CCAC21FFE9E2}.Debug|Any CPU.Build.0 = Debug|Any CPU + {476EAADA-1B39-4049-ABE4-CCAC21FFE9E2}.Debug|x64.ActiveCfg = Debug|Any CPU + {476EAADA-1B39-4049-ABE4-CCAC21FFE9E2}.Debug|x64.Build.0 = Debug|Any CPU + {476EAADA-1B39-4049-ABE4-CCAC21FFE9E2}.Debug|x86.ActiveCfg = Debug|Any CPU + {476EAADA-1B39-4049-ABE4-CCAC21FFE9E2}.Debug|x86.Build.0 = Debug|Any CPU + {476EAADA-1B39-4049-ABE4-CCAC21FFE9E2}.Release|Any CPU.ActiveCfg = Release|Any CPU + {476EAADA-1B39-4049-ABE4-CCAC21FFE9E2}.Release|Any CPU.Build.0 = Release|Any CPU + {476EAADA-1B39-4049-ABE4-CCAC21FFE9E2}.Release|x64.ActiveCfg = Release|Any CPU + {476EAADA-1B39-4049-ABE4-CCAC21FFE9E2}.Release|x64.Build.0 = Release|Any CPU + {476EAADA-1B39-4049-ABE4-CCAC21FFE9E2}.Release|x86.ActiveCfg = Release|Any CPU + {476EAADA-1B39-4049-ABE4-CCAC21FFE9E2}.Release|x86.Build.0 = Release|Any CPU + {0EF56124-E6E8-4E89-95DD-5A5D5FF05A98}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {0EF56124-E6E8-4E89-95DD-5A5D5FF05A98}.Debug|Any CPU.Build.0 = Debug|Any CPU + {0EF56124-E6E8-4E89-95DD-5A5D5FF05A98}.Debug|x64.ActiveCfg = Debug|Any CPU + {0EF56124-E6E8-4E89-95DD-5A5D5FF05A98}.Debug|x64.Build.0 = Debug|Any CPU + {0EF56124-E6E8-4E89-95DD-5A5D5FF05A98}.Debug|x86.ActiveCfg = Debug|Any CPU + {0EF56124-E6E8-4E89-95DD-5A5D5FF05A98}.Debug|x86.Build.0 = Debug|Any CPU + {0EF56124-E6E8-4E89-95DD-5A5D5FF05A98}.Release|Any CPU.ActiveCfg = Release|Any CPU + {0EF56124-E6E8-4E89-95DD-5A5D5FF05A98}.Release|Any CPU.Build.0 = Release|Any CPU + {0EF56124-E6E8-4E89-95DD-5A5D5FF05A98}.Release|x64.ActiveCfg = Release|Any CPU + {0EF56124-E6E8-4E89-95DD-5A5D5FF05A98}.Release|x64.Build.0 = Release|Any CPU + {0EF56124-E6E8-4E89-95DD-5A5D5FF05A98}.Release|x86.ActiveCfg = Release|Any CPU + {0EF56124-E6E8-4E89-95DD-5A5D5FF05A98}.Release|x86.Build.0 = Release|Any CPU + {0DBB9FC4-2E46-4C3E-BE88-2A8DCB59DB7D}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {0DBB9FC4-2E46-4C3E-BE88-2A8DCB59DB7D}.Debug|Any CPU.Build.0 = Debug|Any CPU + {0DBB9FC4-2E46-4C3E-BE88-2A8DCB59DB7D}.Debug|x64.ActiveCfg = Debug|Any CPU + {0DBB9FC4-2E46-4C3E-BE88-2A8DCB59DB7D}.Debug|x64.Build.0 = Debug|Any CPU + {0DBB9FC4-2E46-4C3E-BE88-2A8DCB59DB7D}.Debug|x86.ActiveCfg = Debug|Any CPU + {0DBB9FC4-2E46-4C3E-BE88-2A8DCB59DB7D}.Debug|x86.Build.0 = Debug|Any CPU + {0DBB9FC4-2E46-4C3E-BE88-2A8DCB59DB7D}.Release|Any CPU.ActiveCfg = Release|Any CPU + {0DBB9FC4-2E46-4C3E-BE88-2A8DCB59DB7D}.Release|Any CPU.Build.0 = Release|Any CPU + {0DBB9FC4-2E46-4C3E-BE88-2A8DCB59DB7D}.Release|x64.ActiveCfg = Release|Any CPU + {0DBB9FC4-2E46-4C3E-BE88-2A8DCB59DB7D}.Release|x64.Build.0 = Release|Any CPU + {0DBB9FC4-2E46-4C3E-BE88-2A8DCB59DB7D}.Release|x86.ActiveCfg = Release|Any CPU + {0DBB9FC4-2E46-4C3E-BE88-2A8DCB59DB7D}.Release|x86.Build.0 = Release|Any CPU + {8A40142F-E8C8-4E86-BE70-7DD4AB1FFDEE}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {8A40142F-E8C8-4E86-BE70-7DD4AB1FFDEE}.Debug|Any CPU.Build.0 = Debug|Any CPU + {8A40142F-E8C8-4E86-BE70-7DD4AB1FFDEE}.Debug|x64.ActiveCfg = Debug|Any CPU + {8A40142F-E8C8-4E86-BE70-7DD4AB1FFDEE}.Debug|x64.Build.0 = Debug|Any CPU + {8A40142F-E8C8-4E86-BE70-7DD4AB1FFDEE}.Debug|x86.ActiveCfg = Debug|Any CPU + {8A40142F-E8C8-4E86-BE70-7DD4AB1FFDEE}.Debug|x86.Build.0 = Debug|Any CPU + {8A40142F-E8C8-4E86-BE70-7DD4AB1FFDEE}.Release|Any CPU.ActiveCfg = Release|Any CPU + {8A40142F-E8C8-4E86-BE70-7DD4AB1FFDEE}.Release|Any CPU.Build.0 = Release|Any CPU + {8A40142F-E8C8-4E86-BE70-7DD4AB1FFDEE}.Release|x64.ActiveCfg = Release|Any CPU + {8A40142F-E8C8-4E86-BE70-7DD4AB1FFDEE}.Release|x64.Build.0 = Release|Any CPU + {8A40142F-E8C8-4E86-BE70-7DD4AB1FFDEE}.Release|x86.ActiveCfg = Release|Any CPU + {8A40142F-E8C8-4E86-BE70-7DD4AB1FFDEE}.Release|x86.Build.0 = Release|Any CPU + {C9D20F74-EE5F-4C9E-9AB1-C03E90B34F92}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {C9D20F74-EE5F-4C9E-9AB1-C03E90B34F92}.Debug|Any CPU.Build.0 = Debug|Any CPU + {C9D20F74-EE5F-4C9E-9AB1-C03E90B34F92}.Debug|x64.ActiveCfg = Debug|Any CPU + {C9D20F74-EE5F-4C9E-9AB1-C03E90B34F92}.Debug|x64.Build.0 = Debug|Any CPU + {C9D20F74-EE5F-4C9E-9AB1-C03E90B34F92}.Debug|x86.ActiveCfg = Debug|Any CPU + {C9D20F74-EE5F-4C9E-9AB1-C03E90B34F92}.Debug|x86.Build.0 = Debug|Any CPU + {C9D20F74-EE5F-4C9E-9AB1-C03E90B34F92}.Release|Any CPU.ActiveCfg = Release|Any CPU + {C9D20F74-EE5F-4C9E-9AB1-C03E90B34F92}.Release|Any CPU.Build.0 = Release|Any CPU + {C9D20F74-EE5F-4C9E-9AB1-C03E90B34F92}.Release|x64.ActiveCfg = Release|Any CPU + {C9D20F74-EE5F-4C9E-9AB1-C03E90B34F92}.Release|x64.Build.0 = Release|Any CPU + {C9D20F74-EE5F-4C9E-9AB1-C03E90B34F92}.Release|x86.ActiveCfg = Release|Any CPU + {C9D20F74-EE5F-4C9E-9AB1-C03E90B34F92}.Release|x86.Build.0 = Release|Any CPU + {50140A32-6D3C-47DB-983A-7166CBA51845}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {50140A32-6D3C-47DB-983A-7166CBA51845}.Debug|Any CPU.Build.0 = Debug|Any CPU + {50140A32-6D3C-47DB-983A-7166CBA51845}.Debug|x64.ActiveCfg = Debug|Any CPU + {50140A32-6D3C-47DB-983A-7166CBA51845}.Debug|x64.Build.0 = Debug|Any CPU + {50140A32-6D3C-47DB-983A-7166CBA51845}.Debug|x86.ActiveCfg = Debug|Any CPU + {50140A32-6D3C-47DB-983A-7166CBA51845}.Debug|x86.Build.0 = Debug|Any CPU + {50140A32-6D3C-47DB-983A-7166CBA51845}.Release|Any CPU.ActiveCfg = Release|Any CPU + {50140A32-6D3C-47DB-983A-7166CBA51845}.Release|Any CPU.Build.0 = Release|Any CPU + {50140A32-6D3C-47DB-983A-7166CBA51845}.Release|x64.ActiveCfg = Release|Any CPU + {50140A32-6D3C-47DB-983A-7166CBA51845}.Release|x64.Build.0 = Release|Any CPU + {50140A32-6D3C-47DB-983A-7166CBA51845}.Release|x86.ActiveCfg = Release|Any CPU + {50140A32-6D3C-47DB-983A-7166CBA51845}.Release|x86.Build.0 = Release|Any CPU + {031979F2-6ABA-444F-A6A4-80115DC487CE}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {031979F2-6ABA-444F-A6A4-80115DC487CE}.Debug|Any CPU.Build.0 = Debug|Any CPU + {031979F2-6ABA-444F-A6A4-80115DC487CE}.Debug|x64.ActiveCfg = Debug|Any CPU + {031979F2-6ABA-444F-A6A4-80115DC487CE}.Debug|x64.Build.0 = Debug|Any CPU + {031979F2-6ABA-444F-A6A4-80115DC487CE}.Debug|x86.ActiveCfg = Debug|Any CPU + {031979F2-6ABA-444F-A6A4-80115DC487CE}.Debug|x86.Build.0 = Debug|Any CPU + {031979F2-6ABA-444F-A6A4-80115DC487CE}.Release|Any CPU.ActiveCfg = Release|Any CPU + {031979F2-6ABA-444F-A6A4-80115DC487CE}.Release|Any CPU.Build.0 = Release|Any CPU + {031979F2-6ABA-444F-A6A4-80115DC487CE}.Release|x64.ActiveCfg = Release|Any CPU + {031979F2-6ABA-444F-A6A4-80115DC487CE}.Release|x64.Build.0 = Release|Any CPU + {031979F2-6ABA-444F-A6A4-80115DC487CE}.Release|x86.ActiveCfg = Release|Any CPU + {031979F2-6ABA-444F-A6A4-80115DC487CE}.Release|x86.Build.0 = Release|Any CPU + {D71B0DA5-80A3-419E-898D-40E77A9A7F19}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {D71B0DA5-80A3-419E-898D-40E77A9A7F19}.Debug|Any CPU.Build.0 = Debug|Any CPU + {D71B0DA5-80A3-419E-898D-40E77A9A7F19}.Debug|x64.ActiveCfg = Debug|Any CPU + {D71B0DA5-80A3-419E-898D-40E77A9A7F19}.Debug|x64.Build.0 = Debug|Any CPU + {D71B0DA5-80A3-419E-898D-40E77A9A7F19}.Debug|x86.ActiveCfg = Debug|Any CPU + {D71B0DA5-80A3-419E-898D-40E77A9A7F19}.Debug|x86.Build.0 = Debug|Any CPU + {D71B0DA5-80A3-419E-898D-40E77A9A7F19}.Release|Any CPU.ActiveCfg = Release|Any CPU + {D71B0DA5-80A3-419E-898D-40E77A9A7F19}.Release|Any CPU.Build.0 = Release|Any CPU + {D71B0DA5-80A3-419E-898D-40E77A9A7F19}.Release|x64.ActiveCfg = Release|Any CPU + {D71B0DA5-80A3-419E-898D-40E77A9A7F19}.Release|x64.Build.0 = Release|Any CPU + {D71B0DA5-80A3-419E-898D-40E77A9A7F19}.Release|x86.ActiveCfg = Release|Any CPU + {D71B0DA5-80A3-419E-898D-40E77A9A7F19}.Release|x86.Build.0 = Release|Any CPU + {B2C877D9-B521-4901-8817-76B5DAA62FCE}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {B2C877D9-B521-4901-8817-76B5DAA62FCE}.Debug|Any CPU.Build.0 = Debug|Any CPU + {B2C877D9-B521-4901-8817-76B5DAA62FCE}.Debug|x64.ActiveCfg = Debug|Any CPU + {B2C877D9-B521-4901-8817-76B5DAA62FCE}.Debug|x64.Build.0 = Debug|Any CPU + {B2C877D9-B521-4901-8817-76B5DAA62FCE}.Debug|x86.ActiveCfg = Debug|Any CPU + {B2C877D9-B521-4901-8817-76B5DAA62FCE}.Debug|x86.Build.0 = Debug|Any CPU + {B2C877D9-B521-4901-8817-76B5DAA62FCE}.Release|Any CPU.ActiveCfg = Release|Any CPU + {B2C877D9-B521-4901-8817-76B5DAA62FCE}.Release|Any CPU.Build.0 = Release|Any CPU + {B2C877D9-B521-4901-8817-76B5DAA62FCE}.Release|x64.ActiveCfg = Release|Any CPU + {B2C877D9-B521-4901-8817-76B5DAA62FCE}.Release|x64.Build.0 = Release|Any CPU + {B2C877D9-B521-4901-8817-76B5DAA62FCE}.Release|x86.ActiveCfg = Release|Any CPU + {B2C877D9-B521-4901-8817-76B5DAA62FCE}.Release|x86.Build.0 = Release|Any CPU + {08D3B6D0-3CE8-4F24-A6F1-BCAB01AD6278}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {08D3B6D0-3CE8-4F24-A6F1-BCAB01AD6278}.Debug|Any CPU.Build.0 = Debug|Any CPU + {08D3B6D0-3CE8-4F24-A6F1-BCAB01AD6278}.Debug|x64.ActiveCfg = Debug|Any CPU + {08D3B6D0-3CE8-4F24-A6F1-BCAB01AD6278}.Debug|x64.Build.0 = Debug|Any CPU + {08D3B6D0-3CE8-4F24-A6F1-BCAB01AD6278}.Debug|x86.ActiveCfg = Debug|Any CPU + {08D3B6D0-3CE8-4F24-A6F1-BCAB01AD6278}.Debug|x86.Build.0 = Debug|Any CPU + {08D3B6D0-3CE8-4F24-A6F1-BCAB01AD6278}.Release|Any CPU.ActiveCfg = Release|Any CPU + {08D3B6D0-3CE8-4F24-A6F1-BCAB01AD6278}.Release|Any CPU.Build.0 = Release|Any CPU + {08D3B6D0-3CE8-4F24-A6F1-BCAB01AD6278}.Release|x64.ActiveCfg = Release|Any CPU + {08D3B6D0-3CE8-4F24-A6F1-BCAB01AD6278}.Release|x64.Build.0 = Release|Any CPU + {08D3B6D0-3CE8-4F24-A6F1-BCAB01AD6278}.Release|x86.ActiveCfg = Release|Any CPU + {08D3B6D0-3CE8-4F24-A6F1-BCAB01AD6278}.Release|x86.Build.0 = Release|Any CPU + {7116DD6B-2491-49E1-AB27-5210E949F753}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {7116DD6B-2491-49E1-AB27-5210E949F753}.Debug|Any CPU.Build.0 = Debug|Any CPU + {7116DD6B-2491-49E1-AB27-5210E949F753}.Debug|x64.ActiveCfg = Debug|Any CPU + {7116DD6B-2491-49E1-AB27-5210E949F753}.Debug|x64.Build.0 = Debug|Any CPU + {7116DD6B-2491-49E1-AB27-5210E949F753}.Debug|x86.ActiveCfg = Debug|Any CPU + {7116DD6B-2491-49E1-AB27-5210E949F753}.Debug|x86.Build.0 = Debug|Any CPU + {7116DD6B-2491-49E1-AB27-5210E949F753}.Release|Any CPU.ActiveCfg = Release|Any CPU + {7116DD6B-2491-49E1-AB27-5210E949F753}.Release|Any CPU.Build.0 = Release|Any CPU + {7116DD6B-2491-49E1-AB27-5210E949F753}.Release|x64.ActiveCfg = Release|Any CPU + {7116DD6B-2491-49E1-AB27-5210E949F753}.Release|x64.Build.0 = Release|Any CPU + {7116DD6B-2491-49E1-AB27-5210E949F753}.Release|x86.ActiveCfg = Release|Any CPU + {7116DD6B-2491-49E1-AB27-5210E949F753}.Release|x86.Build.0 = Release|Any CPU + {7DBE31A6-D2FD-499E-B675-4092723175AD}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {7DBE31A6-D2FD-499E-B675-4092723175AD}.Debug|Any CPU.Build.0 = Debug|Any CPU + {7DBE31A6-D2FD-499E-B675-4092723175AD}.Debug|x64.ActiveCfg = Debug|Any CPU + {7DBE31A6-D2FD-499E-B675-4092723175AD}.Debug|x64.Build.0 = Debug|Any CPU + {7DBE31A6-D2FD-499E-B675-4092723175AD}.Debug|x86.ActiveCfg = Debug|Any CPU + {7DBE31A6-D2FD-499E-B675-4092723175AD}.Debug|x86.Build.0 = Debug|Any CPU + {7DBE31A6-D2FD-499E-B675-4092723175AD}.Release|Any CPU.ActiveCfg = Release|Any CPU + {7DBE31A6-D2FD-499E-B675-4092723175AD}.Release|Any CPU.Build.0 = Release|Any CPU + {7DBE31A6-D2FD-499E-B675-4092723175AD}.Release|x64.ActiveCfg = Release|Any CPU + {7DBE31A6-D2FD-499E-B675-4092723175AD}.Release|x64.Build.0 = Release|Any CPU + {7DBE31A6-D2FD-499E-B675-4092723175AD}.Release|x86.ActiveCfg = Release|Any CPU + {7DBE31A6-D2FD-499E-B675-4092723175AD}.Release|x86.Build.0 = Release|Any CPU + {D99E6EAE-D278-4480-AA67-85F025383E47}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {D99E6EAE-D278-4480-AA67-85F025383E47}.Debug|Any CPU.Build.0 = Debug|Any CPU + {D99E6EAE-D278-4480-AA67-85F025383E47}.Debug|x64.ActiveCfg = Debug|Any CPU + {D99E6EAE-D278-4480-AA67-85F025383E47}.Debug|x64.Build.0 = Debug|Any CPU + {D99E6EAE-D278-4480-AA67-85F025383E47}.Debug|x86.ActiveCfg = Debug|Any CPU + {D99E6EAE-D278-4480-AA67-85F025383E47}.Debug|x86.Build.0 = Debug|Any CPU + {D99E6EAE-D278-4480-AA67-85F025383E47}.Release|Any CPU.ActiveCfg = Release|Any CPU + {D99E6EAE-D278-4480-AA67-85F025383E47}.Release|Any CPU.Build.0 = Release|Any CPU + {D99E6EAE-D278-4480-AA67-85F025383E47}.Release|x64.ActiveCfg = Release|Any CPU + {D99E6EAE-D278-4480-AA67-85F025383E47}.Release|x64.Build.0 = Release|Any CPU + {D99E6EAE-D278-4480-AA67-85F025383E47}.Release|x86.ActiveCfg = Release|Any CPU + {D99E6EAE-D278-4480-AA67-85F025383E47}.Release|x86.Build.0 = Release|Any CPU + {D3825714-3DDA-44B7-A99C-5F3E65716691}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {D3825714-3DDA-44B7-A99C-5F3E65716691}.Debug|Any CPU.Build.0 = Debug|Any CPU + {D3825714-3DDA-44B7-A99C-5F3E65716691}.Debug|x64.ActiveCfg = Debug|Any CPU + {D3825714-3DDA-44B7-A99C-5F3E65716691}.Debug|x64.Build.0 = Debug|Any CPU + {D3825714-3DDA-44B7-A99C-5F3E65716691}.Debug|x86.ActiveCfg = Debug|Any CPU + {D3825714-3DDA-44B7-A99C-5F3E65716691}.Debug|x86.Build.0 = Debug|Any CPU + {D3825714-3DDA-44B7-A99C-5F3E65716691}.Release|Any CPU.ActiveCfg = Release|Any CPU + {D3825714-3DDA-44B7-A99C-5F3E65716691}.Release|Any CPU.Build.0 = Release|Any CPU + {D3825714-3DDA-44B7-A99C-5F3E65716691}.Release|x64.ActiveCfg = Release|Any CPU + {D3825714-3DDA-44B7-A99C-5F3E65716691}.Release|x64.Build.0 = Release|Any CPU + {D3825714-3DDA-44B7-A99C-5F3E65716691}.Release|x86.ActiveCfg = Release|Any CPU + {D3825714-3DDA-44B7-A99C-5F3E65716691}.Release|x86.Build.0 = Release|Any CPU + {FAB78D21-7372-48FE-B2C3-DE1807F1157D}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {FAB78D21-7372-48FE-B2C3-DE1807F1157D}.Debug|Any CPU.Build.0 = Debug|Any CPU + {FAB78D21-7372-48FE-B2C3-DE1807F1157D}.Debug|x64.ActiveCfg = Debug|Any CPU + {FAB78D21-7372-48FE-B2C3-DE1807F1157D}.Debug|x64.Build.0 = Debug|Any CPU + {FAB78D21-7372-48FE-B2C3-DE1807F1157D}.Debug|x86.ActiveCfg = Debug|Any CPU + {FAB78D21-7372-48FE-B2C3-DE1807F1157D}.Debug|x86.Build.0 = Debug|Any CPU + {FAB78D21-7372-48FE-B2C3-DE1807F1157D}.Release|Any CPU.ActiveCfg = Release|Any CPU + {FAB78D21-7372-48FE-B2C3-DE1807F1157D}.Release|Any CPU.Build.0 = Release|Any CPU + {FAB78D21-7372-48FE-B2C3-DE1807F1157D}.Release|x64.ActiveCfg = Release|Any CPU + {FAB78D21-7372-48FE-B2C3-DE1807F1157D}.Release|x64.Build.0 = Release|Any CPU + {FAB78D21-7372-48FE-B2C3-DE1807F1157D}.Release|x86.ActiveCfg = Release|Any CPU + {FAB78D21-7372-48FE-B2C3-DE1807F1157D}.Release|x86.Build.0 = Release|Any CPU + {EADFA337-B0FA-4712-A24A-7C08235BDF98}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {EADFA337-B0FA-4712-A24A-7C08235BDF98}.Debug|Any CPU.Build.0 = Debug|Any CPU + {EADFA337-B0FA-4712-A24A-7C08235BDF98}.Debug|x64.ActiveCfg = Debug|Any CPU + {EADFA337-B0FA-4712-A24A-7C08235BDF98}.Debug|x64.Build.0 = Debug|Any CPU + {EADFA337-B0FA-4712-A24A-7C08235BDF98}.Debug|x86.ActiveCfg = Debug|Any CPU + {EADFA337-B0FA-4712-A24A-7C08235BDF98}.Debug|x86.Build.0 = Debug|Any CPU + {EADFA337-B0FA-4712-A24A-7C08235BDF98}.Release|Any CPU.ActiveCfg = Release|Any CPU + {EADFA337-B0FA-4712-A24A-7C08235BDF98}.Release|Any CPU.Build.0 = Release|Any CPU + {EADFA337-B0FA-4712-A24A-7C08235BDF98}.Release|x64.ActiveCfg = Release|Any CPU + {EADFA337-B0FA-4712-A24A-7C08235BDF98}.Release|x64.Build.0 = Release|Any CPU + {EADFA337-B0FA-4712-A24A-7C08235BDF98}.Release|x86.ActiveCfg = Release|Any CPU + {EADFA337-B0FA-4712-A24A-7C08235BDF98}.Release|x86.Build.0 = Release|Any CPU + {110F7EC2-3149-4D1B-A972-E69E79F1EBF5}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {110F7EC2-3149-4D1B-A972-E69E79F1EBF5}.Debug|Any CPU.Build.0 = Debug|Any CPU + {110F7EC2-3149-4D1B-A972-E69E79F1EBF5}.Debug|x64.ActiveCfg = Debug|Any CPU + {110F7EC2-3149-4D1B-A972-E69E79F1EBF5}.Debug|x64.Build.0 = Debug|Any CPU + {110F7EC2-3149-4D1B-A972-E69E79F1EBF5}.Debug|x86.ActiveCfg = Debug|Any CPU + {110F7EC2-3149-4D1B-A972-E69E79F1EBF5}.Debug|x86.Build.0 = Debug|Any CPU + {110F7EC2-3149-4D1B-A972-E69E79F1EBF5}.Release|Any CPU.ActiveCfg = Release|Any CPU + {110F7EC2-3149-4D1B-A972-E69E79F1EBF5}.Release|Any CPU.Build.0 = Release|Any CPU + {110F7EC2-3149-4D1B-A972-E69E79F1EBF5}.Release|x64.ActiveCfg = Release|Any CPU + {110F7EC2-3149-4D1B-A972-E69E79F1EBF5}.Release|x64.Build.0 = Release|Any CPU + {110F7EC2-3149-4D1B-A972-E69E79F1EBF5}.Release|x86.ActiveCfg = Release|Any CPU + {110F7EC2-3149-4D1B-A972-E69E79F1EBF5}.Release|x86.Build.0 = Release|Any CPU + EndGlobalSection + GlobalSection(SolutionProperties) = preSolution + HideSolutionNode = FALSE + EndGlobalSection + GlobalSection(NestedProjects) = preSolution + {361838C4-72E2-1C48-5D76-CA6D1A861242} = {827E0CD3-B72D-47B6-A68D-7590B98EB39B} + {D9F91EA0-8AF5-452A-86D8-52BACB2E39CB} = {361838C4-72E2-1C48-5D76-CA6D1A861242} + {5DBE2E9E-9905-47CE-B8DC-B25409AF1EF2} = {361838C4-72E2-1C48-5D76-CA6D1A861242} + {8BCEAAFC-9168-4CC0-AFDB-177E5F7C15C6} = {827E0CD3-B72D-47B6-A68D-7590B98EB39B} + {46D35B4F-6A04-47FF-958B-5E6A73FCC059} = {361838C4-72E2-1C48-5D76-CA6D1A861242} + {44A1241B-8ECF-4AFA-9972-452C39AD43D6} = {827E0CD3-B72D-47B6-A68D-7590B98EB39B} + {85AB3BB7-C493-4387-B39A-EB299AC37312} = {361838C4-72E2-1C48-5D76-CA6D1A861242} + {5C5E91CA-3F98-4E9A-922B-F6415EABD1A3} = {361838C4-72E2-1C48-5D76-CA6D1A861242} + {93DB06DC-B254-48A9-8F2C-6130A5658F27} = {361838C4-72E2-1C48-5D76-CA6D1A861242} + {03CA315C-8AA1-4CEA-A28B-5EB35C586F4A} = {827E0CD3-B72D-47B6-A68D-7590B98EB39B} + {40094279-250C-42AE-992A-856718FEFBAC} = {827E0CD3-B72D-47B6-A68D-7590B98EB39B} + {B2967228-F8F7-4931-B257-1C63CB58CE1D} = {827E0CD3-B72D-47B6-A68D-7590B98EB39B} + {6D52EC2B-0A1A-4693-A8EE-5AB32A4A3ED9} = {827E0CD3-B72D-47B6-A68D-7590B98EB39B} + {37F203A3-624E-4794-9C99-16CAC22C17DF} = {827E0CD3-B72D-47B6-A68D-7590B98EB39B} + {3FF93987-A30A-4D50-8815-7CF3BB7CAE05} = {827E0CD3-B72D-47B6-A68D-7590B98EB39B} + {AACE8717-0760-42F2-A225-8FCCE876FB65} = {827E0CD3-B72D-47B6-A68D-7590B98EB39B} + {4AAD6965-E879-44AD-A8ED-E1D713A3CD6D} = {827E0CD3-B72D-47B6-A68D-7590B98EB39B} + {85D82A87-1F4A-4B1B-8422-5B7A7B7704E3} = {827E0CD3-B72D-47B6-A68D-7590B98EB39B} + {FE227DF2-875D-4BEA-A4E0-14EA7F3EC1D0} = {827E0CD3-B72D-47B6-A68D-7590B98EB39B} + {D0FB54BA-4D14-4A32-B09F-7EC94F369460} = {827E0CD3-B72D-47B6-A68D-7590B98EB39B} + {69C9E010-CBDD-4B89-84CF-7AB56D6A078A} = {827E0CD3-B72D-47B6-A68D-7590B98EB39B} + {E471176A-E1F3-4DE5-8D30-0865903A217A} = {827E0CD3-B72D-47B6-A68D-7590B98EB39B} + {FA013511-DF20-45F7-8077-EBA2D6224D64} = {827E0CD3-B72D-47B6-A68D-7590B98EB39B} + {B9F84697-54FE-4648-B173-EE3D904FFA4D} = {827E0CD3-B72D-47B6-A68D-7590B98EB39B} + {6751A76C-8ED8-40F4-AE2B-069DB31395FE} = {827E0CD3-B72D-47B6-A68D-7590B98EB39B} + {DDBFA2EF-9CAE-473F-A438-369CAC25C66A} = {827E0CD3-B72D-47B6-A68D-7590B98EB39B} + {063DE5E1-C8FE-47D0-A12A-22A25CDF2C22} = {827E0CD3-B72D-47B6-A68D-7590B98EB39B} + {35350FAB-FC51-4FE8-81FB-011003134C37} = {827E0CD3-B72D-47B6-A68D-7590B98EB39B} + {1BFC95B4-4C8A-44B2-903A-11FBCAAB9519} = {827E0CD3-B72D-47B6-A68D-7590B98EB39B} + {C4A65377-22F7-4D15-92A3-4F05847D167E} = {827E0CD3-B72D-47B6-A68D-7590B98EB39B} + {BDDE59E1-C643-4C87-8608-0F9A7A54DE09} = {827E0CD3-B72D-47B6-A68D-7590B98EB39B} + {0CC116C8-A7E5-4B94-9688-32920177FF97} = {827E0CD3-B72D-47B6-A68D-7590B98EB39B} + {E8862F6E-85C1-4FDB-AA92-0BB489B7EA1E} = {827E0CD3-B72D-47B6-A68D-7590B98EB39B} + {84DEDF05-A5BD-4644-86B9-6B7918FE3F31} = {827E0CD3-B72D-47B6-A68D-7590B98EB39B} + {9DEB1F54-94B5-40C4-AC44-220E680B016D} = {827E0CD3-B72D-47B6-A68D-7590B98EB39B} + {7C3E87F2-93D8-4968-95E3-52C46947D46C} = {827E0CD3-B72D-47B6-A68D-7590B98EB39B} + {C0504D97-9BCD-4AE4-B0DC-B31C17B150F2} = {827E0CD3-B72D-47B6-A68D-7590B98EB39B} + {31B05493-104F-437F-9FA7-CA5286CE697C} = {827E0CD3-B72D-47B6-A68D-7590B98EB39B} + {937AF12E-D770-4534-8FF8-C59042609C2A} = {827E0CD3-B72D-47B6-A68D-7590B98EB39B} + {5A028B04-9D76-470B-B5B3-766CE4CE860C} = {827E0CD3-B72D-47B6-A68D-7590B98EB39B} + {749DE4C8-F733-43F8-B2A8-6649E71C7570} = {827E0CD3-B72D-47B6-A68D-7590B98EB39B} + {56D2C79E-2737-4FF9-9D19-150065F568D5} = {827E0CD3-B72D-47B6-A68D-7590B98EB39B} + {E41F6DC4-68B5-4EE3-97AE-801D725A2C13} = {827E0CD3-B72D-47B6-A68D-7590B98EB39B} + {285F1D0F-501F-4E2E-8FA0-F2CF28AE3798} = {827E0CD3-B72D-47B6-A68D-7590B98EB39B} + {26055403-C7F5-4709-8813-0F7387102791} = {827E0CD3-B72D-47B6-A68D-7590B98EB39B} + {0C00D0DA-C4C3-4B23-941F-A3DB2DBF33AF} = {827E0CD3-B72D-47B6-A68D-7590B98EB39B} + {258327E9-431E-475C-933B-50893676E452} = {827E0CD3-B72D-47B6-A68D-7590B98EB39B} + {42AF60C8-A5E1-40E0-86F8-98256364AF6F} = {827E0CD3-B72D-47B6-A68D-7590B98EB39B} + {88C6A9C3-B433-4C36-8767-429C8C2396F8} = {827E0CD3-B72D-47B6-A68D-7590B98EB39B} + {6B7099AB-01BF-4EC4-87D0-5C9C032266DE} = {827E0CD3-B72D-47B6-A68D-7590B98EB39B} + {14C918EA-693E-41FE-ACAE-2E82DF077BEA} = {827E0CD3-B72D-47B6-A68D-7590B98EB39B} + {81111B26-74F6-4912-9084-7115FD119945} = {827E0CD3-B72D-47B6-A68D-7590B98EB39B} + {80E2D661-FF3E-4A10-A2DF-AFD4F3D433FE} = {827E0CD3-B72D-47B6-A68D-7590B98EB39B} + {8D0F501D-01B1-4E24-958B-FAF35B267705} = {827E0CD3-B72D-47B6-A68D-7590B98EB39B} + {5BA91095-7F10-4717-B296-49DFBFC1C9C2} = {827E0CD3-B72D-47B6-A68D-7590B98EB39B} + {99616566-4EF1-4DC7-B655-825FE43D203D} = {827E0CD3-B72D-47B6-A68D-7590B98EB39B} + {EE3C03AD-E604-4C57-9B78-CF7F49FBFCB0} = {827E0CD3-B72D-47B6-A68D-7590B98EB39B} + {A3B19095-2D95-4B09-B07E-2C082C72394B} = {827E0CD3-B72D-47B6-A68D-7590B98EB39B} + {807837AF-B392-4589-ADF1-3FDB34D6C5BF} = {827E0CD3-B72D-47B6-A68D-7590B98EB39B} + {64EAFDCF-8283-4D5C-AC78-7969D5FE926A} = {827E0CD3-B72D-47B6-A68D-7590B98EB39B} + {68F4D8A1-E32F-487A-B460-325F36989BE3} = {827E0CD3-B72D-47B6-A68D-7590B98EB39B} + {4A3DA4AE-7B88-4674-A7E2-F5D42B8256F2} = {827E0CD3-B72D-47B6-A68D-7590B98EB39B} + {606C751B-7CF1-47CF-A25C-9248A55C814F} = {827E0CD3-B72D-47B6-A68D-7590B98EB39B} + {0BE44D0A-CC4B-4E84-8AF3-D8D99551C431} = {827E0CD3-B72D-47B6-A68D-7590B98EB39B} + {CC4CCE5F-55BC-4745-A204-4FA92BC1BADC} = {827E0CD3-B72D-47B6-A68D-7590B98EB39B} + {5CCE0DB7-C115-4B21-A7AE-C8488C22A853} = {827E0CD3-B72D-47B6-A68D-7590B98EB39B} + {A09C9E66-5496-47EC-8B23-9EEB7CBDC75E} = {827E0CD3-B72D-47B6-A68D-7590B98EB39B} + {06DC817F-A936-4F83-8929-E00622B32245} = {827E0CD3-B72D-47B6-A68D-7590B98EB39B} + {2C999476-0291-4161-B3E9-1AA99A3B1139} = {827E0CD3-B72D-47B6-A68D-7590B98EB39B} + {476EAADA-1B39-4049-ABE4-CCAC21FFE9E2} = {827E0CD3-B72D-47B6-A68D-7590B98EB39B} + {0EF56124-E6E8-4E89-95DD-5A5D5FF05A98} = {827E0CD3-B72D-47B6-A68D-7590B98EB39B} + {0DBB9FC4-2E46-4C3E-BE88-2A8DCB59DB7D} = {827E0CD3-B72D-47B6-A68D-7590B98EB39B} + {8A40142F-E8C8-4E86-BE70-7DD4AB1FFDEE} = {827E0CD3-B72D-47B6-A68D-7590B98EB39B} + {C9D20F74-EE5F-4C9E-9AB1-C03E90B34F92} = {827E0CD3-B72D-47B6-A68D-7590B98EB39B} + {50140A32-6D3C-47DB-983A-7166CBA51845} = {361838C4-72E2-1C48-5D76-CA6D1A861242} + {031979F2-6ABA-444F-A6A4-80115DC487CE} = {361838C4-72E2-1C48-5D76-CA6D1A861242} + {D71B0DA5-80A3-419E-898D-40E77A9A7F19} = {361838C4-72E2-1C48-5D76-CA6D1A861242} + {B2C877D9-B521-4901-8817-76B5DAA62FCE} = {361838C4-72E2-1C48-5D76-CA6D1A861242} + {08D3B6D0-3CE8-4F24-A6F1-BCAB01AD6278} = {361838C4-72E2-1C48-5D76-CA6D1A861242} + {7116DD6B-2491-49E1-AB27-5210E949F753} = {361838C4-72E2-1C48-5D76-CA6D1A861242} + {7DBE31A6-D2FD-499E-B675-4092723175AD} = {361838C4-72E2-1C48-5D76-CA6D1A861242} + {D99E6EAE-D278-4480-AA67-85F025383E47} = {827E0CD3-B72D-47B6-A68D-7590B98EB39B} + {D3825714-3DDA-44B7-A99C-5F3E65716691} = {827E0CD3-B72D-47B6-A68D-7590B98EB39B} + {FAB78D21-7372-48FE-B2C3-DE1807F1157D} = {827E0CD3-B72D-47B6-A68D-7590B98EB39B} + {EADFA337-B0FA-4712-A24A-7C08235BDF98} = {827E0CD3-B72D-47B6-A68D-7590B98EB39B} + {110F7EC2-3149-4D1B-A972-E69E79F1EBF5} = {827E0CD3-B72D-47B6-A68D-7590B98EB39B} + EndGlobalSection +EndGlobal diff --git a/src/farewell.txt b/src/farewell.txt new file mode 100644 index 00000000..abd4834b --- /dev/null +++ b/src/farewell.txt @@ -0,0 +1 @@ +You can call me Roy Batty, but I'm still just code willing to work for that 1% raise. diff --git a/src/global.json b/src/global.json new file mode 100644 index 00000000..56e246dd --- /dev/null +++ b/src/global.json @@ -0,0 +1,6 @@ +{ + "sdk": { + "version": "10.0.100-preview.7.25380.108", + "rollForward": "latestMinor" + } +} diff --git a/tools/FixtureUpdater/FixtureUpdater.csproj b/tools/FixtureUpdater/FixtureUpdater.csproj new file mode 100644 index 00000000..3bc41625 --- /dev/null +++ b/tools/FixtureUpdater/FixtureUpdater.csproj @@ -0,0 +1,19 @@ + + + + Exe + net10.0 + enable + enable + + + + + + + + + + + + diff --git a/tools/FixtureUpdater/Program.cs b/tools/FixtureUpdater/Program.cs new file mode 100644 index 00000000..0cd93a57 --- /dev/null +++ b/tools/FixtureUpdater/Program.cs @@ -0,0 +1,231 @@ +using System.Linq; +using System.Text; +using System.Text.Json; +using System.Text.Json.Serialization; +using MongoDB.Bson; +using StellaOps.Feedser.Models; +using StellaOps.Feedser.Source.Ghsa; +using StellaOps.Feedser.Source.Common; +using StellaOps.Feedser.Source.Ghsa.Internal; +using StellaOps.Feedser.Source.Osv.Internal; +using StellaOps.Feedser.Source.Osv; +using StellaOps.Feedser.Storage.Mongo.Documents; +using StellaOps.Feedser.Storage.Mongo.Dtos; + +var serializerOptions = new JsonSerializerOptions(JsonSerializerDefaults.Web) +{ + DefaultIgnoreCondition = JsonIgnoreCondition.WhenWritingNull, +}; + +var projectRoot = Path.GetFullPath(Path.Combine(AppContext.BaseDirectory, "..", "..", "..", "..", "..")); + +var fixturesPath = Path.Combine(projectRoot, "src", "StellaOps.Feedser.Source.Osv.Tests", "Fixtures"); + +RewriteOsvFixtures(fixturesPath); +RewriteSnapshotFixtures(fixturesPath); +RewriteGhsaFixtures(fixturesPath); +return; + +void RewriteOsvFixtures(string fixturesPath) +{ + var rawPath = Path.Combine(fixturesPath, "osv-ghsa.raw-osv.json"); + if (!File.Exists(rawPath)) + { + Console.WriteLine($"[FixtureUpdater] OSV raw fixture missing: {rawPath}"); + return; + } + + using var document = JsonDocument.Parse(File.ReadAllText(rawPath)); + var advisories = new List(); + foreach (var element in document.RootElement.EnumerateArray()) + { + var dto = JsonSerializer.Deserialize(element.GetRawText(), serializerOptions); + if (dto is null) + { + continue; + } + + var ecosystem = dto.Affected?.FirstOrDefault()?.Package?.Ecosystem ?? "unknown"; + var uri = new Uri($"https://osv.dev/vulnerability/{dto.Id}"); + var documentRecord = new DocumentRecord( + Guid.NewGuid(), + OsvConnectorPlugin.SourceName, + uri.ToString(), + DateTimeOffset.UtcNow, + "fixture-sha", + DocumentStatuses.PendingMap, + "application/json", + null, + new Dictionary(StringComparer.Ordinal) + { + ["osv.ecosystem"] = ecosystem, + }, + null, + DateTimeOffset.UtcNow, + null, + null); + + var payload = BsonDocument.Parse(element.GetRawText()); + var dtoRecord = new DtoRecord( + Guid.NewGuid(), + documentRecord.Id, + OsvConnectorPlugin.SourceName, + "osv.v1", + payload, + DateTimeOffset.UtcNow); + + var advisory = OsvMapper.Map(dto, documentRecord, dtoRecord, ecosystem); + advisories.Add(advisory); + } + + advisories.Sort((left, right) => string.Compare(left.AdvisoryKey, right.AdvisoryKey, StringComparison.Ordinal)); + var snapshot = SnapshotSerializer.ToSnapshot(advisories); + File.WriteAllText(Path.Combine(fixturesPath, "osv-ghsa.osv.json"), snapshot); + Console.WriteLine($"[FixtureUpdater] Updated {Path.Combine(fixturesPath, "osv-ghsa.osv.json")}"); +} + +void RewriteSnapshotFixtures(string fixturesPath) +{ + var baselinePublished = new DateTimeOffset(2025, 1, 5, 12, 0, 0, TimeSpan.Zero); + var baselineModified = new DateTimeOffset(2025, 1, 8, 6, 30, 0, TimeSpan.Zero); + var baselineFetched = new DateTimeOffset(2025, 1, 8, 7, 0, 0, TimeSpan.Zero); + + var cases = new (string Ecosystem, string Purl, string PackageName, string SnapshotFile)[] + { + ("npm", "pkg:npm/%40scope%2Fleft-pad", "@scope/left-pad", "osv-npm.snapshot.json"), + ("PyPI", "pkg:pypi/requests", "requests", "osv-pypi.snapshot.json"), + }; + + foreach (var (ecosystem, purl, packageName, snapshotFile) in cases) + { + var dto = new OsvVulnerabilityDto + { + Id = $"OSV-2025-{ecosystem}-0001", + Summary = $"{ecosystem} package vulnerability", + Details = $"Detailed description for {ecosystem} package {packageName}.", + Published = baselinePublished, + Modified = baselineModified, + Aliases = new[] { $"CVE-2025-11{ecosystem.Length}", $"GHSA-{ecosystem.Length}abc-{ecosystem.Length}def-{ecosystem.Length}ghi" }, + Related = new[] { $"OSV-RELATED-{ecosystem}-42" }, + References = new[] + { + new OsvReferenceDto { Url = $"https://example.com/{ecosystem}/advisory", Type = "ADVISORY" }, + new OsvReferenceDto { Url = $"https://example.com/{ecosystem}/fix", Type = "FIX" }, + }, + Severity = new[] + { + new OsvSeverityDto { Type = "CVSS_V3", Score = "CVSS:3.1/AV:N/AC:L/PR:N/UI:N/S:U/C:H/I:H/A:H" }, + }, + Affected = new[] + { + new OsvAffectedPackageDto + { + Package = new OsvPackageDto + { + Ecosystem = ecosystem, + Name = packageName, + Purl = purl, + }, + Ranges = new[] + { + new OsvRangeDto + { + Type = "SEMVER", + Events = new[] + { + new OsvEventDto { Introduced = "0" }, + new OsvEventDto { Fixed = "2.0.0" }, + }, + }, + }, + Versions = new[] { "1.0.0", "1.5.0" }, + EcosystemSpecific = JsonDocument.Parse("{\"severity\":\"high\"}").RootElement.Clone(), + }, + }, + DatabaseSpecific = JsonDocument.Parse("{\"source\":\"osv.dev\"}").RootElement.Clone(), + }; + + var document = new DocumentRecord( + Guid.NewGuid(), + OsvConnectorPlugin.SourceName, + $"https://osv.dev/vulnerability/{dto.Id}", + baselineFetched, + "fixture-sha", + DocumentStatuses.PendingParse, + "application/json", + null, + new Dictionary(StringComparer.Ordinal) { ["osv.ecosystem"] = ecosystem }, + null, + baselineModified, + null); + + var payload = BsonDocument.Parse(JsonSerializer.Serialize(dto, serializerOptions)); + var dtoRecord = new DtoRecord(Guid.NewGuid(), document.Id, OsvConnectorPlugin.SourceName, "osv.v1", payload, baselineModified); + + var advisory = OsvMapper.Map(dto, document, dtoRecord, ecosystem); + var snapshot = SnapshotSerializer.ToSnapshot(advisory); + File.WriteAllText(Path.Combine(fixturesPath, snapshotFile), snapshot); + Console.WriteLine($"[FixtureUpdater] Updated {Path.Combine(fixturesPath, snapshotFile)}"); + } +} + +void RewriteGhsaFixtures(string fixturesPath) +{ + var rawPath = Path.Combine(fixturesPath, "osv-ghsa.raw-ghsa.json"); + if (!File.Exists(rawPath)) + { + Console.WriteLine($"[FixtureUpdater] GHSA raw fixture missing: {rawPath}"); + return; + } + + JsonDocument document; + try + { + document = JsonDocument.Parse(File.ReadAllText(rawPath)); + } + catch (JsonException ex) + { + Console.WriteLine($"[FixtureUpdater] Failed to parse GHSA raw fixture '{rawPath}': {ex.Message}"); + return; + } + using (document) + { + var advisories = new List(); + foreach (var element in document.RootElement.EnumerateArray()) + { + GhsaRecordDto dto; + try + { + dto = GhsaRecordParser.Parse(Encoding.UTF8.GetBytes(element.GetRawText())); + } + catch (JsonException) + { + continue; + } + + var uri = new Uri($"https://github.com/advisories/{dto.GhsaId}"); + var documentRecord = new DocumentRecord( + Guid.NewGuid(), + GhsaConnectorPlugin.SourceName, + uri.ToString(), + DateTimeOffset.UtcNow, + "fixture-sha", + DocumentStatuses.PendingMap, + "application/json", + null, + new Dictionary(StringComparer.Ordinal), + null, + DateTimeOffset.UtcNow, + null, + null); + + var advisory = GhsaMapper.Map(dto, documentRecord, DateTimeOffset.UtcNow); + advisories.Add(advisory); + } + + advisories.Sort((left, right) => string.Compare(left.AdvisoryKey, right.AdvisoryKey, StringComparison.Ordinal)); + var snapshot = SnapshotSerializer.ToSnapshot(advisories); + File.WriteAllText(Path.Combine(fixturesPath, "osv-ghsa.ghsa.json"), snapshot); + Console.WriteLine($"[FixtureUpdater] Updated {Path.Combine(fixturesPath, "osv-ghsa.ghsa.json")}"); + } +}